feat(reconciler): lab5 recover expired leases and cleanup staging
This commit is contained in:
@@ -103,6 +103,7 @@ func main() {
|
||||
store.Chats(),
|
||||
objectStorage,
|
||||
store.Outbox(),
|
||||
store.FailedJobs(),
|
||||
store,
|
||||
cfg.MaxUploadBytes,
|
||||
cfg.MaxImagePixels,
|
||||
@@ -139,6 +140,7 @@ func main() {
|
||||
reconciler := service.NewReconciler(
|
||||
store.Attachments(),
|
||||
objectStorage,
|
||||
store.Outbox(),
|
||||
pool,
|
||||
cfg.ReconcilerGracePeriod,
|
||||
)
|
||||
|
||||
@@ -41,11 +41,14 @@ type AttachmentRepository interface {
|
||||
GetByID(ctx context.Context, id string) (*Attachment, error)
|
||||
MarkReady(ctx context.Context, id string) error
|
||||
MarkFailed(ctx context.Context, id string, errorCode string) error
|
||||
ListStalePending(ctx context.Context, olderThan time.Time, limit int) ([]Attachment, error)
|
||||
ExistingKeys(ctx context.Context, keys []string) ([]string, error)
|
||||
DeletePending(ctx context.Context, id string) error
|
||||
CreateInTx(ctx context.Context, tx Tx, a *Attachment) error
|
||||
ResetForReplay(ctx context.Context, tx Tx, id string) error
|
||||
|
||||
ListExpiredLeases(ctx context.Context, limit int) ([]Attachment, error)
|
||||
ResetLease(ctx context.Context, id string) error
|
||||
ListStagingToCleanup(ctx context.Context, limit int) ([]Attachment, error)
|
||||
ClearStagingKey(ctx context.Context, id string) error
|
||||
}
|
||||
|
||||
type OutboxRepository interface {
|
||||
|
||||
@@ -2,6 +2,7 @@ package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"log"
|
||||
"time"
|
||||
|
||||
@@ -15,6 +16,7 @@ const advisoryLockKey int64 = 0x4D5455534904
|
||||
type Reconciler struct {
|
||||
attachments domain.AttachmentRepository
|
||||
storage domain.ObjectStorage
|
||||
outbox domain.OutboxRepository
|
||||
pool *pgxpool.Pool
|
||||
gracePeriod time.Duration
|
||||
batchSize int
|
||||
@@ -23,6 +25,7 @@ type Reconciler struct {
|
||||
func NewReconciler(
|
||||
attachments domain.AttachmentRepository,
|
||||
storage domain.ObjectStorage,
|
||||
outbox domain.OutboxRepository,
|
||||
pool *pgxpool.Pool,
|
||||
gracePeriod time.Duration,
|
||||
) *Reconciler {
|
||||
@@ -32,6 +35,7 @@ func NewReconciler(
|
||||
return &Reconciler{
|
||||
attachments: attachments,
|
||||
storage: storage,
|
||||
outbox: outbox,
|
||||
pool: pool,
|
||||
gracePeriod: gracePeriod,
|
||||
batchSize: 100,
|
||||
@@ -45,14 +49,13 @@ func (r *Reconciler) Run(ctx context.Context, interval time.Duration) {
|
||||
ticker := time.NewTicker(interval)
|
||||
defer ticker.Stop()
|
||||
|
||||
log.Printf("reconciler: started, interval=%s grace=%s", interval, r.gracePeriod)
|
||||
|
||||
log.Printf("reconciler: запущен (interval=%s, grace=%s)", interval, r.gracePeriod)
|
||||
r.tick(ctx)
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
log.Println("reconciler: stopped")
|
||||
log.Println("reconciler: остановлен")
|
||||
return
|
||||
case <-ticker.C:
|
||||
r.tick(ctx)
|
||||
@@ -63,7 +66,7 @@ func (r *Reconciler) Run(ctx context.Context, interval time.Duration) {
|
||||
func (r *Reconciler) tick(ctx context.Context) {
|
||||
acquired, release, err := r.acquireLock(ctx)
|
||||
if err != nil {
|
||||
log.Printf("reconciler: acquire lock: %v", err)
|
||||
log.Printf("reconciler: ошибка получения блокировки: %v", err)
|
||||
return
|
||||
}
|
||||
if !acquired {
|
||||
@@ -71,11 +74,12 @@ func (r *Reconciler) tick(ctx context.Context) {
|
||||
}
|
||||
defer release()
|
||||
|
||||
pendingDeleted := r.sweepPending(ctx)
|
||||
leasesReset := r.sweepExpiredLeases(ctx)
|
||||
stagingDeleted := r.sweepStaging(ctx)
|
||||
orphansDeleted := r.sweepOrphans(ctx)
|
||||
|
||||
if pendingDeleted > 0 || orphansDeleted > 0 {
|
||||
log.Printf("reconciler: sweep done pending=%d orphans=%d", pendingDeleted, orphansDeleted)
|
||||
if leasesReset > 0 || stagingDeleted > 0 || orphansDeleted > 0 {
|
||||
log.Printf("reconciler: очистка завершена (восстановлено_задач=%d, удалено_staging=%d, удалено_сирот=%d)", leasesReset, stagingDeleted, orphansDeleted)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -99,29 +103,64 @@ func (r *Reconciler) acquireLock(ctx context.Context) (bool, func(), error) {
|
||||
unlockCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
if _, err := conn.Exec(unlockCtx, `SELECT pg_advisory_unlock($1)`, advisoryLockKey); err != nil {
|
||||
log.Printf("reconciler: unlock: %v", err)
|
||||
log.Printf("reconciler: ошибка снятия блокировки: %v", err)
|
||||
}
|
||||
conn.Release()
|
||||
}
|
||||
return true, release, nil
|
||||
}
|
||||
|
||||
func (r *Reconciler) sweepPending(ctx context.Context) int {
|
||||
cutoff := time.Now().UTC().Add(-r.gracePeriod)
|
||||
stale, err := r.attachments.ListStalePending(ctx, cutoff, r.batchSize)
|
||||
func (r *Reconciler) sweepExpiredLeases(ctx context.Context) int {
|
||||
expired, err := r.attachments.ListExpiredLeases(ctx, r.batchSize)
|
||||
if err != nil {
|
||||
log.Printf("reconciler: list stale pending: %v", err)
|
||||
log.Printf("reconciler: ошибка получения истёкших аренд: %v", err)
|
||||
return 0
|
||||
}
|
||||
|
||||
recovered := 0
|
||||
for _, a := range expired {
|
||||
if err := r.attachments.ResetLease(ctx, a.ID); err != nil {
|
||||
log.Printf("reconciler: ошибка сброса аренды для %s: %v", a.ID, err)
|
||||
continue
|
||||
}
|
||||
|
||||
payload, _ := json.Marshal(domain.JobMessage{
|
||||
AttachmentID: a.ID,
|
||||
PipelineVersion: a.PipelineVersion,
|
||||
Attempt: a.Attempts + 1,
|
||||
})
|
||||
err := r.outbox.EnqueueDirect(ctx, &domain.OutboxEntry{
|
||||
AggregateType: domain.OutboxAggregateAttachment,
|
||||
AggregateID: a.ID,
|
||||
EventType: domain.OutboxEventProcess,
|
||||
Payload: payload,
|
||||
CreatedAt: time.Now().UTC(),
|
||||
})
|
||||
if err != nil {
|
||||
log.Printf("reconciler: ошибка постановки в очередь восстановленной задачи %s: %v", a.ID, err)
|
||||
continue
|
||||
}
|
||||
recovered++
|
||||
}
|
||||
return recovered
|
||||
}
|
||||
|
||||
// Чистим staging в S3 для готовых задач
|
||||
func (r *Reconciler) sweepStaging(ctx context.Context) int {
|
||||
toCleanup, err := r.attachments.ListStagingToCleanup(ctx, r.batchSize)
|
||||
if err != nil {
|
||||
log.Printf("reconciler: ошибка получения списка staging для очистки: %v", err)
|
||||
return 0
|
||||
}
|
||||
|
||||
deleted := 0
|
||||
for _, a := range stale {
|
||||
if err := r.storage.Delete(ctx, a.ObjectKey); err != nil {
|
||||
log.Printf("reconciler: delete object %s: %v", a.ObjectKey, err)
|
||||
for _, a := range toCleanup {
|
||||
if err := r.storage.Delete(ctx, a.StagingKey); err != nil {
|
||||
log.Printf("reconciler: ошибка удаления объекта staging %s: %v", a.StagingKey, err)
|
||||
continue
|
||||
}
|
||||
if err := r.attachments.DeletePending(ctx, a.ID); err != nil {
|
||||
log.Printf("reconciler: delete pending row %s: %v", a.ID, err)
|
||||
if err := r.attachments.ClearStagingKey(ctx, a.ID); err != nil {
|
||||
log.Printf("reconciler: ошибка очистки ключа staging %s: %v", a.ID, err)
|
||||
continue
|
||||
}
|
||||
deleted++
|
||||
@@ -129,10 +168,11 @@ func (r *Reconciler) sweepPending(ctx context.Context) int {
|
||||
return deleted
|
||||
}
|
||||
|
||||
// Удаляем старые сиротские объекты (без записи в БД)
|
||||
func (r *Reconciler) sweepOrphans(ctx context.Context) int {
|
||||
objects, err := r.storage.List(ctx, "chats/")
|
||||
if err != nil {
|
||||
log.Printf("reconciler: list objects: %v", err)
|
||||
log.Printf("reconciler: ошибка получения списка объектов: %v", err)
|
||||
return 0
|
||||
}
|
||||
if len(objects) == 0 {
|
||||
@@ -152,7 +192,7 @@ func (r *Reconciler) sweepOrphans(ctx context.Context) int {
|
||||
|
||||
existing, err := r.attachments.ExistingKeys(ctx, keys)
|
||||
if err != nil {
|
||||
log.Printf("reconciler: existing keys: %v", err)
|
||||
log.Printf("reconciler: ошибка получения существующих ключей: %v", err)
|
||||
return 0
|
||||
}
|
||||
|
||||
@@ -167,7 +207,7 @@ func (r *Reconciler) sweepOrphans(ctx context.Context) int {
|
||||
continue
|
||||
}
|
||||
if err := r.storage.Delete(ctx, k); err != nil {
|
||||
log.Printf("reconciler: delete orphan %s: %v", k, err)
|
||||
log.Printf("reconciler: ошибка удаления сиротского объекта %s: %v", k, err)
|
||||
continue
|
||||
}
|
||||
deleted++
|
||||
|
||||
@@ -173,47 +173,6 @@ func (r *AttachmentRepo) MarkFailed(ctx context.Context, id, errorCode string) e
|
||||
}
|
||||
}
|
||||
|
||||
func (r *AttachmentRepo) ListStalePending(ctx context.Context, olderThan time.Time, limit int) ([]domain.Attachment, error) {
|
||||
if limit <= 0 {
|
||||
return []domain.Attachment{}, nil
|
||||
}
|
||||
const q = `
|
||||
SELECT id, chat_id, owner_id, filename, content_type, size_bytes, sha256,
|
||||
object_key, status, error_code, created_at, updated_at
|
||||
FROM attachments
|
||||
WHERE status = 'pending' AND created_at < $1
|
||||
ORDER BY created_at
|
||||
LIMIT $2
|
||||
`
|
||||
rows, err := r.pool.Query(ctx, q, olderThan, limit)
|
||||
if err != nil {
|
||||
return nil, mapError(err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
out := make([]domain.Attachment, 0, limit)
|
||||
for rows.Next() {
|
||||
var a domain.Attachment
|
||||
var status string
|
||||
if err := rows.Scan(&a.ID, &a.ChatID, &a.OwnerID, &a.Filename,
|
||||
&a.ContentType, &a.SizeBytes, &a.SHA256, &a.ObjectKey,
|
||||
&status, &a.ErrorCode, &a.CreatedAt, &a.UpdatedAt); err != nil {
|
||||
return nil, mapError(err)
|
||||
}
|
||||
a.Status = domain.AttachmentStatus(status)
|
||||
out = append(out, a)
|
||||
}
|
||||
return out, mapError(rows.Err())
|
||||
}
|
||||
|
||||
func (r *AttachmentRepo) DeletePending(ctx context.Context, id string) error {
|
||||
const q = `DELETE FROM attachments WHERE id = $1::uuid AND status = 'pending'`
|
||||
if _, err := r.pool.Exec(ctx, q, id); err != nil {
|
||||
return mapError(err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *AttachmentRepo) ExistingKeys(ctx context.Context, keys []string) ([]string, error) {
|
||||
if len(keys) == 0 {
|
||||
return []string{}, nil
|
||||
@@ -316,4 +275,78 @@ func (r *AttachmentRepo) ResetForReplay(ctx context.Context, tx domain.Tx, id st
|
||||
return tx.Exec(ctx, q, id)
|
||||
}
|
||||
|
||||
// для сверщика.
|
||||
func (r *AttachmentRepo) ListExpiredLeases(ctx context.Context, limit int) ([]domain.Attachment, error) {
|
||||
const q = `
|
||||
SELECT id, chat_id, owner_id, filename, content_type, size_bytes, sha256,
|
||||
object_key, status, error_code, created_at, updated_at,
|
||||
attempts, next_attempt_at, locked_until, pipeline_version, staging_key
|
||||
FROM attachments
|
||||
WHERE status = 'processing' AND locked_until < now()
|
||||
ORDER BY locked_until
|
||||
LIMIT $1
|
||||
`
|
||||
rows, err := r.pool.Query(ctx, q, limit)
|
||||
if err != nil {
|
||||
return nil, mapError(err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var out []domain.Attachment
|
||||
for rows.Next() {
|
||||
var a domain.Attachment
|
||||
var status string
|
||||
err := rows.Scan(&a.ID, &a.ChatID, &a.OwnerID, &a.Filename, &a.ContentType,
|
||||
&a.SizeBytes, &a.SHA256, &a.ObjectKey, &status, &a.ErrorCode,
|
||||
&a.CreatedAt, &a.UpdatedAt, &a.Attempts, &a.NextAttemptAt, &a.LockedUntil,
|
||||
&a.PipelineVersion, &a.StagingKey)
|
||||
if err != nil {
|
||||
return nil, mapError(err)
|
||||
}
|
||||
a.Status = domain.AttachmentStatus(status)
|
||||
out = append(out, a)
|
||||
}
|
||||
return out, mapError(rows.Err())
|
||||
}
|
||||
|
||||
func (r *AttachmentRepo) ResetLease(ctx context.Context, id string) error {
|
||||
const q = `
|
||||
UPDATE attachments
|
||||
SET status = 'queued', locked_until = NULL, updated_at = now()
|
||||
WHERE id = $1::uuid AND status = 'processing' AND locked_until < now()
|
||||
`
|
||||
_, err := r.pool.Exec(ctx, q, id)
|
||||
return mapError(err)
|
||||
}
|
||||
|
||||
func (r *AttachmentRepo) ListStagingToCleanup(ctx context.Context, limit int) ([]domain.Attachment, error) {
|
||||
const q = `
|
||||
SELECT id, staging_key
|
||||
FROM attachments
|
||||
WHERE status = 'ready' AND staging_key IS NOT NULL
|
||||
LIMIT $1
|
||||
`
|
||||
rows, err := r.pool.Query(ctx, q, limit)
|
||||
if err != nil {
|
||||
return nil, mapError(err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var out []domain.Attachment
|
||||
for rows.Next() {
|
||||
var a domain.Attachment
|
||||
if err := rows.Scan(&a.ID, &a.StagingKey); err != nil {
|
||||
return nil, mapError(err)
|
||||
}
|
||||
out = append(out, a)
|
||||
}
|
||||
return out, mapError(rows.Err())
|
||||
}
|
||||
|
||||
func (r *AttachmentRepo) ClearStagingKey(ctx context.Context, id string) error {
|
||||
const q = `UPDATE attachments SET staging_key = NULL WHERE id = $1::uuid`
|
||||
_, err := r.pool.Exec(ctx, q, id)
|
||||
return mapError(err)
|
||||
}
|
||||
|
||||
var _ domain.AttachmentRepository = (*AttachmentRepo)(nil)
|
||||
|
||||
Reference in New Issue
Block a user