From d40b339906c0f5232ceac01dcd77d6592b151096 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=93=D0=BE=D0=BB=D0=B8=D0=BA=D0=BE=D0=B2=20Maximillian?= =?UTF-8?q?=20=D0=90=D0=BD=D1=82=D0=BE=D0=BD?= Date: Wed, 11 Nov 2026 12:40:53 +0300 Subject: [PATCH] feat(reconciler): lab5 recover expired leases and cleanup staging --- backend/cmd/server/main.go | 2 + backend/internal/domain/repository.go | 7 +- backend/internal/service/reconciler.go | 80 +++++++++--- .../internal/storage/postgres/attachments.go | 115 +++++++++++------- 4 files changed, 141 insertions(+), 63 deletions(-) diff --git a/backend/cmd/server/main.go b/backend/cmd/server/main.go index 66ab20a..ffba9c0 100644 --- a/backend/cmd/server/main.go +++ b/backend/cmd/server/main.go @@ -103,6 +103,7 @@ func main() { store.Chats(), objectStorage, store.Outbox(), + store.FailedJobs(), store, cfg.MaxUploadBytes, cfg.MaxImagePixels, @@ -139,6 +140,7 @@ func main() { reconciler := service.NewReconciler( store.Attachments(), objectStorage, + store.Outbox(), pool, cfg.ReconcilerGracePeriod, ) diff --git a/backend/internal/domain/repository.go b/backend/internal/domain/repository.go index dc07f3a..66d5142 100644 --- a/backend/internal/domain/repository.go +++ b/backend/internal/domain/repository.go @@ -41,11 +41,14 @@ type AttachmentRepository interface { GetByID(ctx context.Context, id string) (*Attachment, error) MarkReady(ctx context.Context, id string) error MarkFailed(ctx context.Context, id string, errorCode string) error - ListStalePending(ctx context.Context, olderThan time.Time, limit int) ([]Attachment, error) ExistingKeys(ctx context.Context, keys []string) ([]string, error) - DeletePending(ctx context.Context, id string) error CreateInTx(ctx context.Context, tx Tx, a *Attachment) error ResetForReplay(ctx context.Context, tx Tx, id string) error + + ListExpiredLeases(ctx context.Context, limit int) ([]Attachment, error) + ResetLease(ctx context.Context, id string) error + ListStagingToCleanup(ctx context.Context, limit int) ([]Attachment, error) + ClearStagingKey(ctx context.Context, id string) error } type OutboxRepository interface { diff --git a/backend/internal/service/reconciler.go b/backend/internal/service/reconciler.go index 325b95d..a1c69c0 100644 --- a/backend/internal/service/reconciler.go +++ b/backend/internal/service/reconciler.go @@ -2,6 +2,7 @@ package service import ( "context" + "encoding/json" "log" "time" @@ -15,6 +16,7 @@ const advisoryLockKey int64 = 0x4D5455534904 type Reconciler struct { attachments domain.AttachmentRepository storage domain.ObjectStorage + outbox domain.OutboxRepository pool *pgxpool.Pool gracePeriod time.Duration batchSize int @@ -23,6 +25,7 @@ type Reconciler struct { func NewReconciler( attachments domain.AttachmentRepository, storage domain.ObjectStorage, + outbox domain.OutboxRepository, pool *pgxpool.Pool, gracePeriod time.Duration, ) *Reconciler { @@ -32,6 +35,7 @@ func NewReconciler( return &Reconciler{ attachments: attachments, storage: storage, + outbox: outbox, pool: pool, gracePeriod: gracePeriod, batchSize: 100, @@ -45,14 +49,13 @@ func (r *Reconciler) Run(ctx context.Context, interval time.Duration) { ticker := time.NewTicker(interval) defer ticker.Stop() - log.Printf("reconciler: started, interval=%s grace=%s", interval, r.gracePeriod) - + log.Printf("reconciler: запущен (interval=%s, grace=%s)", interval, r.gracePeriod) r.tick(ctx) for { select { case <-ctx.Done(): - log.Println("reconciler: stopped") + log.Println("reconciler: остановлен") return case <-ticker.C: r.tick(ctx) @@ -63,7 +66,7 @@ func (r *Reconciler) Run(ctx context.Context, interval time.Duration) { func (r *Reconciler) tick(ctx context.Context) { acquired, release, err := r.acquireLock(ctx) if err != nil { - log.Printf("reconciler: acquire lock: %v", err) + log.Printf("reconciler: ошибка получения блокировки: %v", err) return } if !acquired { @@ -71,11 +74,12 @@ func (r *Reconciler) tick(ctx context.Context) { } defer release() - pendingDeleted := r.sweepPending(ctx) + leasesReset := r.sweepExpiredLeases(ctx) + stagingDeleted := r.sweepStaging(ctx) orphansDeleted := r.sweepOrphans(ctx) - if pendingDeleted > 0 || orphansDeleted > 0 { - log.Printf("reconciler: sweep done pending=%d orphans=%d", pendingDeleted, orphansDeleted) + if leasesReset > 0 || stagingDeleted > 0 || orphansDeleted > 0 { + log.Printf("reconciler: очистка завершена (восстановлено_задач=%d, удалено_staging=%d, удалено_сирот=%d)", leasesReset, stagingDeleted, orphansDeleted) } } @@ -99,29 +103,64 @@ func (r *Reconciler) acquireLock(ctx context.Context) (bool, func(), error) { unlockCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() if _, err := conn.Exec(unlockCtx, `SELECT pg_advisory_unlock($1)`, advisoryLockKey); err != nil { - log.Printf("reconciler: unlock: %v", err) + log.Printf("reconciler: ошибка снятия блокировки: %v", err) } conn.Release() } return true, release, nil } -func (r *Reconciler) sweepPending(ctx context.Context) int { - cutoff := time.Now().UTC().Add(-r.gracePeriod) - stale, err := r.attachments.ListStalePending(ctx, cutoff, r.batchSize) +func (r *Reconciler) sweepExpiredLeases(ctx context.Context) int { + expired, err := r.attachments.ListExpiredLeases(ctx, r.batchSize) if err != nil { - log.Printf("reconciler: list stale pending: %v", err) + log.Printf("reconciler: ошибка получения истёкших аренд: %v", err) + return 0 + } + + recovered := 0 + for _, a := range expired { + if err := r.attachments.ResetLease(ctx, a.ID); err != nil { + log.Printf("reconciler: ошибка сброса аренды для %s: %v", a.ID, err) + continue + } + + payload, _ := json.Marshal(domain.JobMessage{ + AttachmentID: a.ID, + PipelineVersion: a.PipelineVersion, + Attempt: a.Attempts + 1, + }) + err := r.outbox.EnqueueDirect(ctx, &domain.OutboxEntry{ + AggregateType: domain.OutboxAggregateAttachment, + AggregateID: a.ID, + EventType: domain.OutboxEventProcess, + Payload: payload, + CreatedAt: time.Now().UTC(), + }) + if err != nil { + log.Printf("reconciler: ошибка постановки в очередь восстановленной задачи %s: %v", a.ID, err) + continue + } + recovered++ + } + return recovered +} + +// Чистим staging в S3 для готовых задач +func (r *Reconciler) sweepStaging(ctx context.Context) int { + toCleanup, err := r.attachments.ListStagingToCleanup(ctx, r.batchSize) + if err != nil { + log.Printf("reconciler: ошибка получения списка staging для очистки: %v", err) return 0 } deleted := 0 - for _, a := range stale { - if err := r.storage.Delete(ctx, a.ObjectKey); err != nil { - log.Printf("reconciler: delete object %s: %v", a.ObjectKey, err) + for _, a := range toCleanup { + if err := r.storage.Delete(ctx, a.StagingKey); err != nil { + log.Printf("reconciler: ошибка удаления объекта staging %s: %v", a.StagingKey, err) continue } - if err := r.attachments.DeletePending(ctx, a.ID); err != nil { - log.Printf("reconciler: delete pending row %s: %v", a.ID, err) + if err := r.attachments.ClearStagingKey(ctx, a.ID); err != nil { + log.Printf("reconciler: ошибка очистки ключа staging %s: %v", a.ID, err) continue } deleted++ @@ -129,10 +168,11 @@ func (r *Reconciler) sweepPending(ctx context.Context) int { return deleted } +// Удаляем старые сиротские объекты (без записи в БД) func (r *Reconciler) sweepOrphans(ctx context.Context) int { objects, err := r.storage.List(ctx, "chats/") if err != nil { - log.Printf("reconciler: list objects: %v", err) + log.Printf("reconciler: ошибка получения списка объектов: %v", err) return 0 } if len(objects) == 0 { @@ -152,7 +192,7 @@ func (r *Reconciler) sweepOrphans(ctx context.Context) int { existing, err := r.attachments.ExistingKeys(ctx, keys) if err != nil { - log.Printf("reconciler: existing keys: %v", err) + log.Printf("reconciler: ошибка получения существующих ключей: %v", err) return 0 } @@ -167,7 +207,7 @@ func (r *Reconciler) sweepOrphans(ctx context.Context) int { continue } if err := r.storage.Delete(ctx, k); err != nil { - log.Printf("reconciler: delete orphan %s: %v", k, err) + log.Printf("reconciler: ошибка удаления сиротского объекта %s: %v", k, err) continue } deleted++ diff --git a/backend/internal/storage/postgres/attachments.go b/backend/internal/storage/postgres/attachments.go index 3289216..9c77802 100644 --- a/backend/internal/storage/postgres/attachments.go +++ b/backend/internal/storage/postgres/attachments.go @@ -173,47 +173,6 @@ func (r *AttachmentRepo) MarkFailed(ctx context.Context, id, errorCode string) e } } -func (r *AttachmentRepo) ListStalePending(ctx context.Context, olderThan time.Time, limit int) ([]domain.Attachment, error) { - if limit <= 0 { - return []domain.Attachment{}, nil - } - const q = ` - SELECT id, chat_id, owner_id, filename, content_type, size_bytes, sha256, - object_key, status, error_code, created_at, updated_at - FROM attachments - WHERE status = 'pending' AND created_at < $1 - ORDER BY created_at - LIMIT $2 - ` - rows, err := r.pool.Query(ctx, q, olderThan, limit) - if err != nil { - return nil, mapError(err) - } - defer rows.Close() - - out := make([]domain.Attachment, 0, limit) - for rows.Next() { - var a domain.Attachment - var status string - if err := rows.Scan(&a.ID, &a.ChatID, &a.OwnerID, &a.Filename, - &a.ContentType, &a.SizeBytes, &a.SHA256, &a.ObjectKey, - &status, &a.ErrorCode, &a.CreatedAt, &a.UpdatedAt); err != nil { - return nil, mapError(err) - } - a.Status = domain.AttachmentStatus(status) - out = append(out, a) - } - return out, mapError(rows.Err()) -} - -func (r *AttachmentRepo) DeletePending(ctx context.Context, id string) error { - const q = `DELETE FROM attachments WHERE id = $1::uuid AND status = 'pending'` - if _, err := r.pool.Exec(ctx, q, id); err != nil { - return mapError(err) - } - return nil -} - func (r *AttachmentRepo) ExistingKeys(ctx context.Context, keys []string) ([]string, error) { if len(keys) == 0 { return []string{}, nil @@ -316,4 +275,78 @@ func (r *AttachmentRepo) ResetForReplay(ctx context.Context, tx domain.Tx, id st return tx.Exec(ctx, q, id) } +// для сверщика. +func (r *AttachmentRepo) ListExpiredLeases(ctx context.Context, limit int) ([]domain.Attachment, error) { + const q = ` + SELECT id, chat_id, owner_id, filename, content_type, size_bytes, sha256, + object_key, status, error_code, created_at, updated_at, + attempts, next_attempt_at, locked_until, pipeline_version, staging_key + FROM attachments + WHERE status = 'processing' AND locked_until < now() + ORDER BY locked_until + LIMIT $1 + ` + rows, err := r.pool.Query(ctx, q, limit) + if err != nil { + return nil, mapError(err) + } + defer rows.Close() + + var out []domain.Attachment + for rows.Next() { + var a domain.Attachment + var status string + err := rows.Scan(&a.ID, &a.ChatID, &a.OwnerID, &a.Filename, &a.ContentType, + &a.SizeBytes, &a.SHA256, &a.ObjectKey, &status, &a.ErrorCode, + &a.CreatedAt, &a.UpdatedAt, &a.Attempts, &a.NextAttemptAt, &a.LockedUntil, + &a.PipelineVersion, &a.StagingKey) + if err != nil { + return nil, mapError(err) + } + a.Status = domain.AttachmentStatus(status) + out = append(out, a) + } + return out, mapError(rows.Err()) +} + +func (r *AttachmentRepo) ResetLease(ctx context.Context, id string) error { + const q = ` + UPDATE attachments + SET status = 'queued', locked_until = NULL, updated_at = now() + WHERE id = $1::uuid AND status = 'processing' AND locked_until < now() + ` + _, err := r.pool.Exec(ctx, q, id) + return mapError(err) +} + +func (r *AttachmentRepo) ListStagingToCleanup(ctx context.Context, limit int) ([]domain.Attachment, error) { + const q = ` + SELECT id, staging_key + FROM attachments + WHERE status = 'ready' AND staging_key IS NOT NULL + LIMIT $1 + ` + rows, err := r.pool.Query(ctx, q, limit) + if err != nil { + return nil, mapError(err) + } + defer rows.Close() + + var out []domain.Attachment + for rows.Next() { + var a domain.Attachment + if err := rows.Scan(&a.ID, &a.StagingKey); err != nil { + return nil, mapError(err) + } + out = append(out, a) + } + return out, mapError(rows.Err()) +} + +func (r *AttachmentRepo) ClearStagingKey(ctx context.Context, id string) error { + const q = `UPDATE attachments SET staging_key = NULL WHERE id = $1::uuid` + _, err := r.pool.Exec(ctx, q, id) + return mapError(err) +} + var _ domain.AttachmentRepository = (*AttachmentRepo)(nil)