forked from mtusi2026/mtusi2026lab5
- RabbitMQ quorum queue, persistent messages and publisher confirms - S3 staging plus PostgreSQL attachments/jobs outbox transaction - worker variants (thumbnail, crop, optimized WebP), DB lease and lease sweeper - jittered retry, permanent failures, explicit replay and admission cap - deterministic crash hooks, resilience script and evidence for grade 4 - async-aware custom tests, oversized pixel case, docs and changelog
111 lines
3.2 KiB
Go
111 lines
3.2 KiB
Go
// Package reconcile repairs the two storage/metadata gaps that a crash can
|
|
// leave behind:
|
|
//
|
|
// - a staging object whose metadata row never committed;
|
|
// - a terminal attachment whose staging copy was not deleted yet.
|
|
//
|
|
// It never touches the final original or a variant while PostgreSQL still owns
|
|
// it. The grace period protects in-flight work.
|
|
package reconcile
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"log/slog"
|
|
"time"
|
|
|
|
"gitflic.ru/project/bulla-max/mtusi2026lab/internal/objectstore"
|
|
"gitflic.ru/project/bulla-max/mtusi2026lab/internal/store"
|
|
)
|
|
|
|
var prefixes = []string{"attachments/", "staging/"}
|
|
|
|
type Reconciler struct {
|
|
store *store.Store
|
|
objects *objectstore.Client
|
|
grace time.Duration
|
|
logger *slog.Logger
|
|
}
|
|
|
|
type Stats struct {
|
|
StaleStaging int `json:"stale_staging"`
|
|
OrphanObjects int `json:"orphan_objects"`
|
|
DeletedStaging int `json:"deleted_staging"`
|
|
DeletedObjects int `json:"deleted_objects"`
|
|
Skipped int `json:"skipped"`
|
|
}
|
|
|
|
func New(st *store.Store, objects *objectstore.Client, grace time.Duration, logger *slog.Logger) *Reconciler {
|
|
if logger == nil {
|
|
logger = slog.Default()
|
|
}
|
|
return &Reconciler{store: st, objects: objects, grace: grace, logger: logger}
|
|
}
|
|
|
|
func (r *Reconciler) RunOnce(ctx context.Context, dryRun bool) (Stats, error) {
|
|
var stats Stats
|
|
cutoff := time.Now().UTC().Add(-r.grace)
|
|
|
|
// Phase 1: ready/failed rows eventually lose their staging copy. The
|
|
// worker usually deletes it immediately after ready commits; this covers a
|
|
// crash or a transient S3 error.
|
|
stale, err := r.store.ListStaleStagingAttachments(ctx, cutoff, 500)
|
|
if err != nil {
|
|
return stats, err
|
|
}
|
|
for _, attachment := range stale {
|
|
stats.StaleStaging++
|
|
if dryRun {
|
|
r.logger.Info("reconcile: would remove stale staging",
|
|
"attachment_id", attachment.ID, "staging_key", attachment.StagingKey)
|
|
continue
|
|
}
|
|
if err = r.objects.Delete(ctx, attachment.StagingKey); err != nil && !errors.Is(err, objectstore.ErrNotFound) {
|
|
r.logger.Error("reconcile: delete staging",
|
|
"err", err, "attachment_id", attachment.ID)
|
|
stats.Skipped++
|
|
continue
|
|
}
|
|
if err = r.store.ClearStagingKey(ctx, attachment.ID); err != nil {
|
|
r.logger.Error("reconcile: clear staging key", "err", err, "attachment_id", attachment.ID)
|
|
stats.Skipped++
|
|
continue
|
|
}
|
|
stats.DeletedStaging++
|
|
}
|
|
|
|
// Phase 2: anything under our prefixes with no database owner. The API
|
|
// writes staging before the metadata commit, so this window is real.
|
|
known, err := r.store.ListStorageKeys(ctx)
|
|
if err != nil {
|
|
return stats, err
|
|
}
|
|
for _, prefix := range prefixes {
|
|
listed, listErr := r.objects.List(ctx, prefix)
|
|
if listErr != nil {
|
|
return stats, listErr
|
|
}
|
|
for _, object := range listed {
|
|
if _, ok := known[object.Key]; ok {
|
|
continue
|
|
}
|
|
if object.LastModified.IsZero() || object.LastModified.After(cutoff) {
|
|
continue
|
|
}
|
|
stats.OrphanObjects++
|
|
if dryRun {
|
|
r.logger.Info("reconcile: would delete orphan object", "object_key", object.Key)
|
|
continue
|
|
}
|
|
if err = r.objects.Delete(ctx, object.Key); err != nil && !errors.Is(err, objectstore.ErrNotFound) {
|
|
r.logger.Error("reconcile: delete orphan object", "err", err, "object_key", object.Key)
|
|
stats.Skipped++
|
|
continue
|
|
}
|
|
stats.DeletedObjects++
|
|
}
|
|
}
|
|
|
|
return stats, nil
|
|
}
|