feat(reconciler): lab5 recover expired leases and cleanup staging

This commit is contained in:
Голиков Maximillian Антон
2026-11-11 12:40:53 +03:00
parent f748c09e10
commit d40b339906
4 changed files with 141 additions and 63 deletions
+2
View File
@@ -103,6 +103,7 @@ func main() {
store.Chats(),
objectStorage,
store.Outbox(),
store.FailedJobs(),
store,
cfg.MaxUploadBytes,
cfg.MaxImagePixels,
@@ -139,6 +140,7 @@ func main() {
reconciler := service.NewReconciler(
store.Attachments(),
objectStorage,
store.Outbox(),
pool,
cfg.ReconcilerGracePeriod,
)
+5 -2
View File
@@ -41,11 +41,14 @@ type AttachmentRepository interface {
GetByID(ctx context.Context, id string) (*Attachment, error)
MarkReady(ctx context.Context, id string) error
MarkFailed(ctx context.Context, id string, errorCode string) error
ListStalePending(ctx context.Context, olderThan time.Time, limit int) ([]Attachment, error)
ExistingKeys(ctx context.Context, keys []string) ([]string, error)
DeletePending(ctx context.Context, id string) error
CreateInTx(ctx context.Context, tx Tx, a *Attachment) error
ResetForReplay(ctx context.Context, tx Tx, id string) error
ListExpiredLeases(ctx context.Context, limit int) ([]Attachment, error)
ResetLease(ctx context.Context, id string) error
ListStagingToCleanup(ctx context.Context, limit int) ([]Attachment, error)
ClearStagingKey(ctx context.Context, id string) error
}
type OutboxRepository interface {
+60 -20
View File
@@ -2,6 +2,7 @@ package service
import (
"context"
"encoding/json"
"log"
"time"
@@ -15,6 +16,7 @@ const advisoryLockKey int64 = 0x4D5455534904
type Reconciler struct {
attachments domain.AttachmentRepository
storage domain.ObjectStorage
outbox domain.OutboxRepository
pool *pgxpool.Pool
gracePeriod time.Duration
batchSize int
@@ -23,6 +25,7 @@ type Reconciler struct {
func NewReconciler(
attachments domain.AttachmentRepository,
storage domain.ObjectStorage,
outbox domain.OutboxRepository,
pool *pgxpool.Pool,
gracePeriod time.Duration,
) *Reconciler {
@@ -32,6 +35,7 @@ func NewReconciler(
return &Reconciler{
attachments: attachments,
storage: storage,
outbox: outbox,
pool: pool,
gracePeriod: gracePeriod,
batchSize: 100,
@@ -45,14 +49,13 @@ func (r *Reconciler) Run(ctx context.Context, interval time.Duration) {
ticker := time.NewTicker(interval)
defer ticker.Stop()
log.Printf("reconciler: started, interval=%s grace=%s", interval, r.gracePeriod)
log.Printf("reconciler: запущен (interval=%s, grace=%s)", interval, r.gracePeriod)
r.tick(ctx)
for {
select {
case <-ctx.Done():
log.Println("reconciler: stopped")
log.Println("reconciler: остановлен")
return
case <-ticker.C:
r.tick(ctx)
@@ -63,7 +66,7 @@ func (r *Reconciler) Run(ctx context.Context, interval time.Duration) {
func (r *Reconciler) tick(ctx context.Context) {
acquired, release, err := r.acquireLock(ctx)
if err != nil {
log.Printf("reconciler: acquire lock: %v", err)
log.Printf("reconciler: ошибка получения блокировки: %v", err)
return
}
if !acquired {
@@ -71,11 +74,12 @@ func (r *Reconciler) tick(ctx context.Context) {
}
defer release()
pendingDeleted := r.sweepPending(ctx)
leasesReset := r.sweepExpiredLeases(ctx)
stagingDeleted := r.sweepStaging(ctx)
orphansDeleted := r.sweepOrphans(ctx)
if pendingDeleted > 0 || orphansDeleted > 0 {
log.Printf("reconciler: sweep done pending=%d orphans=%d", pendingDeleted, orphansDeleted)
if leasesReset > 0 || stagingDeleted > 0 || orphansDeleted > 0 {
log.Printf("reconciler: очистка завершена (восстановлено_задач=%d, удалено_staging=%d, удалено_сирот=%d)", leasesReset, stagingDeleted, orphansDeleted)
}
}
@@ -99,29 +103,64 @@ func (r *Reconciler) acquireLock(ctx context.Context) (bool, func(), error) {
unlockCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
if _, err := conn.Exec(unlockCtx, `SELECT pg_advisory_unlock($1)`, advisoryLockKey); err != nil {
log.Printf("reconciler: unlock: %v", err)
log.Printf("reconciler: ошибка снятия блокировки: %v", err)
}
conn.Release()
}
return true, release, nil
}
func (r *Reconciler) sweepPending(ctx context.Context) int {
cutoff := time.Now().UTC().Add(-r.gracePeriod)
stale, err := r.attachments.ListStalePending(ctx, cutoff, r.batchSize)
func (r *Reconciler) sweepExpiredLeases(ctx context.Context) int {
expired, err := r.attachments.ListExpiredLeases(ctx, r.batchSize)
if err != nil {
log.Printf("reconciler: list stale pending: %v", err)
log.Printf("reconciler: ошибка получения истёкших аренд: %v", err)
return 0
}
recovered := 0
for _, a := range expired {
if err := r.attachments.ResetLease(ctx, a.ID); err != nil {
log.Printf("reconciler: ошибка сброса аренды для %s: %v", a.ID, err)
continue
}
payload, _ := json.Marshal(domain.JobMessage{
AttachmentID: a.ID,
PipelineVersion: a.PipelineVersion,
Attempt: a.Attempts + 1,
})
err := r.outbox.EnqueueDirect(ctx, &domain.OutboxEntry{
AggregateType: domain.OutboxAggregateAttachment,
AggregateID: a.ID,
EventType: domain.OutboxEventProcess,
Payload: payload,
CreatedAt: time.Now().UTC(),
})
if err != nil {
log.Printf("reconciler: ошибка постановки в очередь восстановленной задачи %s: %v", a.ID, err)
continue
}
recovered++
}
return recovered
}
// Чистим staging в S3 для готовых задач
func (r *Reconciler) sweepStaging(ctx context.Context) int {
toCleanup, err := r.attachments.ListStagingToCleanup(ctx, r.batchSize)
if err != nil {
log.Printf("reconciler: ошибка получения списка staging для очистки: %v", err)
return 0
}
deleted := 0
for _, a := range stale {
if err := r.storage.Delete(ctx, a.ObjectKey); err != nil {
log.Printf("reconciler: delete object %s: %v", a.ObjectKey, err)
for _, a := range toCleanup {
if err := r.storage.Delete(ctx, a.StagingKey); err != nil {
log.Printf("reconciler: ошибка удаления объекта staging %s: %v", a.StagingKey, err)
continue
}
if err := r.attachments.DeletePending(ctx, a.ID); err != nil {
log.Printf("reconciler: delete pending row %s: %v", a.ID, err)
if err := r.attachments.ClearStagingKey(ctx, a.ID); err != nil {
log.Printf("reconciler: ошибка очистки ключа staging %s: %v", a.ID, err)
continue
}
deleted++
@@ -129,10 +168,11 @@ func (r *Reconciler) sweepPending(ctx context.Context) int {
return deleted
}
// Удаляем старые сиротские объекты (без записи в БД)
func (r *Reconciler) sweepOrphans(ctx context.Context) int {
objects, err := r.storage.List(ctx, "chats/")
if err != nil {
log.Printf("reconciler: list objects: %v", err)
log.Printf("reconciler: ошибка получения списка объектов: %v", err)
return 0
}
if len(objects) == 0 {
@@ -152,7 +192,7 @@ func (r *Reconciler) sweepOrphans(ctx context.Context) int {
existing, err := r.attachments.ExistingKeys(ctx, keys)
if err != nil {
log.Printf("reconciler: existing keys: %v", err)
log.Printf("reconciler: ошибка получения существующих ключей: %v", err)
return 0
}
@@ -167,7 +207,7 @@ func (r *Reconciler) sweepOrphans(ctx context.Context) int {
continue
}
if err := r.storage.Delete(ctx, k); err != nil {
log.Printf("reconciler: delete orphan %s: %v", k, err)
log.Printf("reconciler: ошибка удаления сиротского объекта %s: %v", k, err)
continue
}
deleted++
@@ -173,47 +173,6 @@ func (r *AttachmentRepo) MarkFailed(ctx context.Context, id, errorCode string) e
}
}
func (r *AttachmentRepo) ListStalePending(ctx context.Context, olderThan time.Time, limit int) ([]domain.Attachment, error) {
if limit <= 0 {
return []domain.Attachment{}, nil
}
const q = `
SELECT id, chat_id, owner_id, filename, content_type, size_bytes, sha256,
object_key, status, error_code, created_at, updated_at
FROM attachments
WHERE status = 'pending' AND created_at < $1
ORDER BY created_at
LIMIT $2
`
rows, err := r.pool.Query(ctx, q, olderThan, limit)
if err != nil {
return nil, mapError(err)
}
defer rows.Close()
out := make([]domain.Attachment, 0, limit)
for rows.Next() {
var a domain.Attachment
var status string
if err := rows.Scan(&a.ID, &a.ChatID, &a.OwnerID, &a.Filename,
&a.ContentType, &a.SizeBytes, &a.SHA256, &a.ObjectKey,
&status, &a.ErrorCode, &a.CreatedAt, &a.UpdatedAt); err != nil {
return nil, mapError(err)
}
a.Status = domain.AttachmentStatus(status)
out = append(out, a)
}
return out, mapError(rows.Err())
}
func (r *AttachmentRepo) DeletePending(ctx context.Context, id string) error {
const q = `DELETE FROM attachments WHERE id = $1::uuid AND status = 'pending'`
if _, err := r.pool.Exec(ctx, q, id); err != nil {
return mapError(err)
}
return nil
}
func (r *AttachmentRepo) ExistingKeys(ctx context.Context, keys []string) ([]string, error) {
if len(keys) == 0 {
return []string{}, nil
@@ -316,4 +275,78 @@ func (r *AttachmentRepo) ResetForReplay(ctx context.Context, tx domain.Tx, id st
return tx.Exec(ctx, q, id)
}
// для сверщика.
func (r *AttachmentRepo) ListExpiredLeases(ctx context.Context, limit int) ([]domain.Attachment, error) {
const q = `
SELECT id, chat_id, owner_id, filename, content_type, size_bytes, sha256,
object_key, status, error_code, created_at, updated_at,
attempts, next_attempt_at, locked_until, pipeline_version, staging_key
FROM attachments
WHERE status = 'processing' AND locked_until < now()
ORDER BY locked_until
LIMIT $1
`
rows, err := r.pool.Query(ctx, q, limit)
if err != nil {
return nil, mapError(err)
}
defer rows.Close()
var out []domain.Attachment
for rows.Next() {
var a domain.Attachment
var status string
err := rows.Scan(&a.ID, &a.ChatID, &a.OwnerID, &a.Filename, &a.ContentType,
&a.SizeBytes, &a.SHA256, &a.ObjectKey, &status, &a.ErrorCode,
&a.CreatedAt, &a.UpdatedAt, &a.Attempts, &a.NextAttemptAt, &a.LockedUntil,
&a.PipelineVersion, &a.StagingKey)
if err != nil {
return nil, mapError(err)
}
a.Status = domain.AttachmentStatus(status)
out = append(out, a)
}
return out, mapError(rows.Err())
}
func (r *AttachmentRepo) ResetLease(ctx context.Context, id string) error {
const q = `
UPDATE attachments
SET status = 'queued', locked_until = NULL, updated_at = now()
WHERE id = $1::uuid AND status = 'processing' AND locked_until < now()
`
_, err := r.pool.Exec(ctx, q, id)
return mapError(err)
}
func (r *AttachmentRepo) ListStagingToCleanup(ctx context.Context, limit int) ([]domain.Attachment, error) {
const q = `
SELECT id, staging_key
FROM attachments
WHERE status = 'ready' AND staging_key IS NOT NULL
LIMIT $1
`
rows, err := r.pool.Query(ctx, q, limit)
if err != nil {
return nil, mapError(err)
}
defer rows.Close()
var out []domain.Attachment
for rows.Next() {
var a domain.Attachment
if err := rows.Scan(&a.ID, &a.StagingKey); err != nil {
return nil, mapError(err)
}
out = append(out, a)
}
return out, mapError(rows.Err())
}
func (r *AttachmentRepo) ClearStagingKey(ctx context.Context, id string) error {
const q = `UPDATE attachments SET staging_key = NULL WHERE id = $1::uuid`
_, err := r.pool.Exec(ctx, q, id)
return mapError(err)
}
var _ domain.AttachmentRepository = (*AttachmentRepo)(nil)