forked from mtusi2026/mtusi2026lab4
feat: two-store reconciler with safe repair and cost measurement (grade 5, track 2)
This commit is contained in:
+81
-8
@@ -5,8 +5,11 @@ import (
|
||||
"encoding/json"
|
||||
"flag"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
"uuid"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
@@ -17,20 +20,31 @@ import (
|
||||
"messenger/internal/repository/s3"
|
||||
)
|
||||
|
||||
type countingTransport struct {
|
||||
base http.RoundTripper
|
||||
count atomic.Int64
|
||||
}
|
||||
|
||||
func (t *countingTransport) RoundTrip(request *http.Request) (*http.Response, error) {
|
||||
t.count.Add(1)
|
||||
return t.base.RoundTrip(request)
|
||||
}
|
||||
|
||||
func main() {
|
||||
dryRun := flag.Bool("dry-run", true, "report only, do not delete orphan objects")
|
||||
dryRun := flag.Bool("dry-run", true, "report only, do not delete or repair anything")
|
||||
grace := flag.Duration("grace", time.Hour, "minimum object age before it can be deleted")
|
||||
prefix := flag.String("prefix", "chats/", "object key prefix to scan")
|
||||
limit := flag.Int("limit", 10000, "maximum number of objects to scan")
|
||||
repairMissing := flag.Bool("repair-missing", true, "delete metadata rows whose object is gone and that no message references")
|
||||
flag.Parse()
|
||||
|
||||
if err := run(*dryRun, *grace, *prefix, *limit); err != nil {
|
||||
if err := run(*dryRun, *grace, *prefix, *limit, *repairMissing); err != nil {
|
||||
fmt.Fprintln(os.Stderr, "reconcile failed:", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
func run(dryRun bool, grace time.Duration, prefix string, limit int) error {
|
||||
func run(dryRun bool, grace time.Duration, prefix string, limit int, repairMissing bool) error {
|
||||
if grace < 0 {
|
||||
return fmt.Errorf("grace must not be negative")
|
||||
}
|
||||
@@ -55,10 +69,11 @@ func run(dryRun bool, grace time.Duration, prefix string, limit int) error {
|
||||
return err
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
|
||||
defer cancel()
|
||||
|
||||
store, err := s3.New(endpoint, region, accessKeyID, secretAccessKey, bucket)
|
||||
transport := &countingTransport{base: http.DefaultTransport}
|
||||
store, err := s3.NewWithTransport(endpoint, region, accessKeyID, secretAccessKey, bucket, transport)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -77,8 +92,12 @@ func run(dryRun bool, grace time.Duration, prefix string, limit int) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
referenced, err := loadReferences(ctx, pool)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
orphans, missing := reconcile.Plan(toObjects(scanned), rows, prefix, time.Now(), grace)
|
||||
orphans, missing := reconcile.Plan(toObjects(scanned), rows, referenced, prefix, time.Now(), grace)
|
||||
if truncated {
|
||||
missing = nil
|
||||
}
|
||||
@@ -93,16 +112,50 @@ func run(dryRun bool, grace time.Duration, prefix string, limit int) error {
|
||||
Orphans: orphans,
|
||||
Missing: missing,
|
||||
}
|
||||
dbQueries := 2
|
||||
|
||||
if !dryRun {
|
||||
for _, orphan := range orphans {
|
||||
if err := store.Delete(ctx, orphan.Key); err != nil {
|
||||
report.DeleteFailures++
|
||||
continue
|
||||
}
|
||||
report.Deleted++
|
||||
report.DeletedObjects++
|
||||
}
|
||||
if repairMissing {
|
||||
for _, item := range missing {
|
||||
if item.Referenced {
|
||||
report.RetainedMissing++
|
||||
continue
|
||||
}
|
||||
id, err := uuid.Parse(item.ID)
|
||||
if err != nil {
|
||||
report.DeleteFailures++
|
||||
continue
|
||||
}
|
||||
tag, err := pool.Exec(ctx, `DELETE FROM attachments WHERE id = $1`, id)
|
||||
dbQueries++
|
||||
if err != nil {
|
||||
report.DeleteFailures++
|
||||
continue
|
||||
}
|
||||
report.DeletedMetadata += int(tag.RowsAffected())
|
||||
}
|
||||
} else {
|
||||
for _, item := range missing {
|
||||
if item.Referenced {
|
||||
report.RetainedMissing++
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
report.S3Requests = transport.count.Load()
|
||||
report.DBQueries = dbQueries
|
||||
report.DurationMS = time.Since(started).Milliseconds()
|
||||
if report.DurationMS > 0 {
|
||||
report.ObjectsPerSecond = float64(report.ScannedObjects) / (float64(report.DurationMS) / 1000)
|
||||
}
|
||||
|
||||
encoder := json.NewEncoder(os.Stdout)
|
||||
encoder.SetIndent("", " ")
|
||||
@@ -110,7 +163,7 @@ func run(dryRun bool, grace time.Duration, prefix string, limit int) error {
|
||||
return err
|
||||
}
|
||||
if report.DeleteFailures > 0 {
|
||||
return fmt.Errorf("%d orphan deletions failed", report.DeleteFailures)
|
||||
return fmt.Errorf("%d repair operations failed", report.DeleteFailures)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -127,6 +180,26 @@ func loadMetadata(ctx context.Context, pool *pgxpool.Pool) ([]reconcile.Metadata
|
||||
})
|
||||
}
|
||||
|
||||
func loadReferences(ctx context.Context, pool *pgxpool.Pool) (map[uuid.UUID]bool, error) {
|
||||
rows, err := pool.Query(ctx, `SELECT DISTINCT attachment_id FROM message_attachments`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ids, err := pgx.CollectRows(rows, func(row pgx.CollectableRow) (uuid.UUID, error) {
|
||||
var id uuid.UUID
|
||||
err := row.Scan(&id)
|
||||
return id, err
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
referenced := make(map[uuid.UUID]bool, len(ids))
|
||||
for _, id := range ids {
|
||||
referenced[id] = true
|
||||
}
|
||||
return referenced, nil
|
||||
}
|
||||
|
||||
func toObjects(scanned []s3.ObjectInfo) []reconcile.Object {
|
||||
objects := make([]reconcile.Object, len(scanned))
|
||||
for i, info := range scanned {
|
||||
|
||||
+47
-1
@@ -6,6 +6,8 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"strconv"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"messenger/internal/config"
|
||||
@@ -79,6 +81,22 @@ func run(command string, args []string) error {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("removed %s\n", args[0])
|
||||
case "seed":
|
||||
if len(args) != 3 {
|
||||
usage()
|
||||
}
|
||||
count, err := strconv.Atoi(args[1])
|
||||
if err != nil || count <= 0 {
|
||||
return fmt.Errorf("count must be a positive integer")
|
||||
}
|
||||
size, err := strconv.Atoi(args[2])
|
||||
if err != nil || size <= 0 {
|
||||
return fmt.Errorf("size must be a positive integer")
|
||||
}
|
||||
if err := seed(ctx, store, args[0], count, size); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("seeded %d objects under %s\n", count, args[0])
|
||||
default:
|
||||
usage()
|
||||
}
|
||||
@@ -105,8 +123,36 @@ func newStore() (*s3.Store, error) {
|
||||
accessKeyID, secretAccessKey, getenv("S3_BUCKET", "messenger-attachments"))
|
||||
}
|
||||
|
||||
func seed(ctx context.Context, store *s3.Store, prefix string, count, size int) error {
|
||||
content := make([]byte, size)
|
||||
workers := 8
|
||||
work := make(chan int)
|
||||
errors := make(chan error, workers)
|
||||
var group sync.WaitGroup
|
||||
for i := 0; i < workers; i++ {
|
||||
group.Add(1)
|
||||
go func() {
|
||||
defer group.Done()
|
||||
for index := range work {
|
||||
key := fmt.Sprintf("%s/%06d", prefix, index)
|
||||
if err := store.Put(ctx, key, bytes.NewReader(content), int64(size), "application/octet-stream"); err != nil {
|
||||
errors <- err
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
for index := 0; index < count; index++ {
|
||||
work <- index
|
||||
}
|
||||
close(work)
|
||||
group.Wait()
|
||||
close(errors)
|
||||
return <-errors
|
||||
}
|
||||
|
||||
func usage() {
|
||||
fmt.Fprintln(os.Stderr, "usage: s3tool put <key> <file> | delete <key> | list [prefix] | buckets | remove-bucket <name>")
|
||||
fmt.Fprintln(os.Stderr, "usage: s3tool put <key> <file> | delete <key> | list [prefix] | buckets | remove-bucket <name> | seed <prefix> <count> <bytes>")
|
||||
os.Exit(2)
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user