Files
Mikhail Verkovykh 9e43faa536 feat(lab5): durable async attachment pipeline through grade 4
- RabbitMQ quorum queue, persistent messages and publisher confirms
- S3 staging plus PostgreSQL attachments/jobs outbox transaction
- worker variants (thumbnail, crop, optimized WebP), DB lease and lease sweeper
- jittered retry, permanent failures, explicit replay and admission cap
- deterministic crash hooks, resilience script and evidence for grade 4
- async-aware custom tests, oversized pixel case, docs and changelog
2026-11-15 00:30:36 +03:00

111 lines
3.2 KiB
Go

// Package reconcile repairs the two storage/metadata gaps that a crash can
// leave behind:
//
// - a staging object whose metadata row never committed;
// - a terminal attachment whose staging copy was not deleted yet.
//
// It never touches the final original or a variant while PostgreSQL still owns
// it. The grace period protects in-flight work.
package reconcile
import (
"context"
"errors"
"log/slog"
"time"
"gitflic.ru/project/bulla-max/mtusi2026lab/internal/objectstore"
"gitflic.ru/project/bulla-max/mtusi2026lab/internal/store"
)
var prefixes = []string{"attachments/", "staging/"}
type Reconciler struct {
store *store.Store
objects *objectstore.Client
grace time.Duration
logger *slog.Logger
}
type Stats struct {
StaleStaging int `json:"stale_staging"`
OrphanObjects int `json:"orphan_objects"`
DeletedStaging int `json:"deleted_staging"`
DeletedObjects int `json:"deleted_objects"`
Skipped int `json:"skipped"`
}
func New(st *store.Store, objects *objectstore.Client, grace time.Duration, logger *slog.Logger) *Reconciler {
if logger == nil {
logger = slog.Default()
}
return &Reconciler{store: st, objects: objects, grace: grace, logger: logger}
}
func (r *Reconciler) RunOnce(ctx context.Context, dryRun bool) (Stats, error) {
var stats Stats
cutoff := time.Now().UTC().Add(-r.grace)
// Phase 1: ready/failed rows eventually lose their staging copy. The
// worker usually deletes it immediately after ready commits; this covers a
// crash or a transient S3 error.
stale, err := r.store.ListStaleStagingAttachments(ctx, cutoff, 500)
if err != nil {
return stats, err
}
for _, attachment := range stale {
stats.StaleStaging++
if dryRun {
r.logger.Info("reconcile: would remove stale staging",
"attachment_id", attachment.ID, "staging_key", attachment.StagingKey)
continue
}
if err = r.objects.Delete(ctx, attachment.StagingKey); err != nil && !errors.Is(err, objectstore.ErrNotFound) {
r.logger.Error("reconcile: delete staging",
"err", err, "attachment_id", attachment.ID)
stats.Skipped++
continue
}
if err = r.store.ClearStagingKey(ctx, attachment.ID); err != nil {
r.logger.Error("reconcile: clear staging key", "err", err, "attachment_id", attachment.ID)
stats.Skipped++
continue
}
stats.DeletedStaging++
}
// Phase 2: anything under our prefixes with no database owner. The API
// writes staging before the metadata commit, so this window is real.
known, err := r.store.ListStorageKeys(ctx)
if err != nil {
return stats, err
}
for _, prefix := range prefixes {
listed, listErr := r.objects.List(ctx, prefix)
if listErr != nil {
return stats, listErr
}
for _, object := range listed {
if _, ok := known[object.Key]; ok {
continue
}
if object.LastModified.IsZero() || object.LastModified.After(cutoff) {
continue
}
stats.OrphanObjects++
if dryRun {
r.logger.Info("reconcile: would delete orphan object", "object_key", object.Key)
continue
}
if err = r.objects.Delete(ctx, object.Key); err != nil && !errors.Is(err, objectstore.ErrNotFound) {
r.logger.Error("reconcile: delete orphan object", "err", err, "object_key", object.Key)
stats.Skipped++
continue
}
stats.DeletedObjects++
}
}
return stats, nil
}