forked from mtusi2026/mtusi2026lab4
116 lines
3.5 KiB
Go
116 lines
3.5 KiB
Go
// Package reconcile repairs the two failure windows between PostgreSQL
|
|
// metadata and S3 bytes:
|
|
//
|
|
// - a pending metadata row whose upload never finished or whose 201 was
|
|
// never returned;
|
|
// - an S3 object whose metadata transaction never committed.
|
|
//
|
|
// Both repairs use a grace period so an upload that is still in flight is
|
|
// never touched.
|
|
package reconcile
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"log/slog"
|
|
"time"
|
|
|
|
"gitflic.ru/project/bulla-max/mtusi2026lab/internal/objectstore"
|
|
"gitflic.ru/project/bulla-max/mtusi2026lab/internal/store"
|
|
)
|
|
|
|
const objectPrefix = "attachments/"
|
|
|
|
type Reconciler struct {
|
|
store *store.Store
|
|
objects *objectstore.Client
|
|
grace time.Duration
|
|
logger *slog.Logger
|
|
}
|
|
|
|
type Stats struct {
|
|
StalePending int `json:"stale_pending"`
|
|
OrphanObjects int `json:"orphan_objects"`
|
|
DeletedRows int `json:"deleted_rows"`
|
|
DeletedObjects int `json:"deleted_objects"`
|
|
Skipped int `json:"skipped"`
|
|
}
|
|
|
|
func New(st *store.Store, objects *objectstore.Client, grace time.Duration, logger *slog.Logger) *Reconciler {
|
|
if logger == nil {
|
|
logger = slog.Default()
|
|
}
|
|
return &Reconciler{store: st, objects: objects, grace: grace, logger: logger}
|
|
}
|
|
|
|
func (r *Reconciler) RunOnce(ctx context.Context, dryRun bool) (Stats, error) {
|
|
var stats Stats
|
|
cutoff := time.Now().UTC().Add(-r.grace)
|
|
|
|
// Phase 1: rows that were committed as pending but never reached ready.
|
|
stale, err := r.store.ListStalePendingAttachments(ctx, cutoff, 500)
|
|
if err != nil {
|
|
return stats, err
|
|
}
|
|
for _, attachment := range stale {
|
|
stats.StalePending++
|
|
if dryRun {
|
|
r.logger.Info("reconcile: would remove stale pending attachment",
|
|
"attachment_id", attachment.ID, "object_key", attachment.ObjectKey,
|
|
"created_at", attachment.CreatedAt)
|
|
continue
|
|
}
|
|
|
|
if err = r.objects.Delete(ctx, attachment.ObjectKey); err != nil && !errors.Is(err, objectstore.ErrNotFound) {
|
|
// Keep the row for the next pass. Deleting it now would lose the
|
|
// only pointer to an object that might still exist.
|
|
r.logger.Error("reconcile: delete stale object",
|
|
"err", err, "attachment_id", attachment.ID)
|
|
stats.Skipped++
|
|
continue
|
|
}
|
|
if err = r.store.DeletePendingAttachment(ctx, attachment.ID); err != nil {
|
|
r.logger.Error("reconcile: delete stale row", "err", err, "attachment_id", attachment.ID)
|
|
stats.Skipped++
|
|
continue
|
|
}
|
|
stats.DeletedRows++
|
|
stats.DeletedObjects++
|
|
}
|
|
|
|
// Phase 2: objects with no metadata row at all. The pending row is written
|
|
// before the PUT in the normal path, so this only happens after a crash in
|
|
// a narrow window (or manual bucket tampering).
|
|
listed, err := r.objects.List(ctx, objectPrefix)
|
|
if err != nil {
|
|
return stats, err
|
|
}
|
|
known, err := r.store.ListAttachmentObjectKeys(ctx)
|
|
if err != nil {
|
|
return stats, err
|
|
}
|
|
for _, object := range listed {
|
|
if _, ok := known[object.Key]; ok {
|
|
continue
|
|
}
|
|
// A missing LastModified must not be treated as ancient: skipping is
|
|
// safer than deleting an object that may have just been created.
|
|
if object.LastModified.IsZero() || object.LastModified.After(cutoff) {
|
|
continue
|
|
}
|
|
stats.OrphanObjects++
|
|
if dryRun {
|
|
r.logger.Info("reconcile: would delete orphan object", "object_key", object.Key)
|
|
continue
|
|
}
|
|
if err = r.objects.Delete(ctx, object.Key); err != nil && !errors.Is(err, objectstore.ErrNotFound) {
|
|
r.logger.Error("reconcile: delete orphan object", "err", err, "object_key", object.Key)
|
|
stats.Skipped++
|
|
continue
|
|
}
|
|
stats.DeletedObjects++
|
|
}
|
|
|
|
return stats, nil
|
|
}
|