forked from mtusi2026/mtusi2026lab4
217 lines
5.5 KiB
Go
217 lines
5.5 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"flag"
|
|
"fmt"
|
|
"net/http"
|
|
"os"
|
|
"sync/atomic"
|
|
"time"
|
|
"uuid"
|
|
|
|
"github.com/jackc/pgx/v5"
|
|
"github.com/jackc/pgx/v5/pgxpool"
|
|
|
|
"messenger/internal/config"
|
|
"messenger/internal/reconcile"
|
|
"messenger/internal/repository/postgres"
|
|
"messenger/internal/repository/s3"
|
|
)
|
|
|
|
type countingTransport struct {
|
|
base http.RoundTripper
|
|
count atomic.Int64
|
|
}
|
|
|
|
func (t *countingTransport) RoundTrip(request *http.Request) (*http.Response, error) {
|
|
t.count.Add(1)
|
|
return t.base.RoundTrip(request)
|
|
}
|
|
|
|
func main() {
|
|
dryRun := flag.Bool("dry-run", true, "report only, do not delete or repair anything")
|
|
grace := flag.Duration("grace", time.Hour, "minimum object age before it can be deleted")
|
|
prefix := flag.String("prefix", "chats/", "object key prefix to scan")
|
|
limit := flag.Int("limit", 10000, "maximum number of objects to scan")
|
|
repairMissing := flag.Bool("repair-missing", true, "delete metadata rows whose object is gone and that no message references")
|
|
flag.Parse()
|
|
|
|
if err := run(*dryRun, *grace, *prefix, *limit, *repairMissing); err != nil {
|
|
fmt.Fprintln(os.Stderr, "reconcile failed:", err)
|
|
os.Exit(1)
|
|
}
|
|
}
|
|
|
|
func run(dryRun bool, grace time.Duration, prefix string, limit int, repairMissing bool) error {
|
|
if grace < 0 {
|
|
return fmt.Errorf("grace must not be negative")
|
|
}
|
|
if limit <= 0 {
|
|
return fmt.Errorf("limit must be positive")
|
|
}
|
|
started := time.Now()
|
|
|
|
endpoint := getenv("S3_ENDPOINT", "http://s3:9000")
|
|
region := getenv("S3_REGION", "us-east-1")
|
|
bucket := getenv("S3_BUCKET", "messenger-attachments")
|
|
accessKeyID, err := config.ReadSecret("S3_ACCESS_KEY_ID_FILE", "/run/secrets/s3_access_key_id")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
secretAccessKey, err := config.ReadSecret("S3_SECRET_ACCESS_KEY_FILE", "/run/secrets/s3_secret_access_key")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
databaseURL, err := config.ReadSecret("DATABASE_URL_FILE", "/run/secrets/database_url")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
|
|
defer cancel()
|
|
|
|
transport := &countingTransport{base: http.DefaultTransport}
|
|
store, err := s3.NewWithTransport(endpoint, region, accessKeyID, secretAccessKey, bucket, transport)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
scanned, truncated, err := store.List(ctx, prefix, limit)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
pool, err := postgres.NewPool(ctx, databaseURL)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer pool.Close()
|
|
|
|
rows, err := loadMetadata(ctx, pool)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
referenced, err := loadReferences(ctx, pool)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
orphans, missing := reconcile.Plan(toObjects(scanned), rows, referenced, prefix, time.Now(), grace)
|
|
if truncated {
|
|
missing = nil
|
|
}
|
|
|
|
report := reconcile.Report{
|
|
Prefix: prefix,
|
|
GraceSeconds: int64(grace.Seconds()),
|
|
DryRun: dryRun,
|
|
ScannedObjects: len(scanned),
|
|
ScannedRows: len(rows),
|
|
Truncated: truncated,
|
|
Orphans: orphans,
|
|
Missing: missing,
|
|
}
|
|
dbQueries := 2
|
|
|
|
if !dryRun {
|
|
for _, orphan := range orphans {
|
|
if err := store.Delete(ctx, orphan.Key); err != nil {
|
|
report.DeleteFailures++
|
|
continue
|
|
}
|
|
report.DeletedObjects++
|
|
}
|
|
if repairMissing {
|
|
for _, item := range missing {
|
|
if item.Referenced {
|
|
report.RetainedMissing++
|
|
continue
|
|
}
|
|
id, err := uuid.Parse(item.ID)
|
|
if err != nil {
|
|
report.DeleteFailures++
|
|
continue
|
|
}
|
|
tag, err := pool.Exec(ctx, `DELETE FROM attachments WHERE id = $1`, id)
|
|
dbQueries++
|
|
if err != nil {
|
|
report.DeleteFailures++
|
|
continue
|
|
}
|
|
report.DeletedMetadata += int(tag.RowsAffected())
|
|
}
|
|
} else {
|
|
for _, item := range missing {
|
|
if item.Referenced {
|
|
report.RetainedMissing++
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
report.S3Requests = transport.count.Load()
|
|
report.DBQueries = dbQueries
|
|
report.DurationMS = time.Since(started).Milliseconds()
|
|
if report.DurationMS > 0 {
|
|
report.ObjectsPerSecond = float64(report.ScannedObjects) / (float64(report.DurationMS) / 1000)
|
|
}
|
|
|
|
encoder := json.NewEncoder(os.Stdout)
|
|
encoder.SetIndent("", " ")
|
|
if err := encoder.Encode(report); err != nil {
|
|
return err
|
|
}
|
|
if report.DeleteFailures > 0 {
|
|
return fmt.Errorf("%d repair operations failed", report.DeleteFailures)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func loadMetadata(ctx context.Context, pool *pgxpool.Pool) ([]reconcile.Metadata, error) {
|
|
rows, err := pool.Query(ctx, `SELECT id, object_key, chat_id, created_at FROM attachments`)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return pgx.CollectRows(rows, func(row pgx.CollectableRow) (reconcile.Metadata, error) {
|
|
var metadata reconcile.Metadata
|
|
err := row.Scan(&metadata.ID, &metadata.ObjectKey, &metadata.ChatID, &metadata.CreatedAt)
|
|
return metadata, err
|
|
})
|
|
}
|
|
|
|
func loadReferences(ctx context.Context, pool *pgxpool.Pool) (map[uuid.UUID]bool, error) {
|
|
rows, err := pool.Query(ctx, `SELECT DISTINCT attachment_id FROM message_attachments`)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
ids, err := pgx.CollectRows(rows, func(row pgx.CollectableRow) (uuid.UUID, error) {
|
|
var id uuid.UUID
|
|
err := row.Scan(&id)
|
|
return id, err
|
|
})
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
referenced := make(map[uuid.UUID]bool, len(ids))
|
|
for _, id := range ids {
|
|
referenced[id] = true
|
|
}
|
|
return referenced, nil
|
|
}
|
|
|
|
func toObjects(scanned []s3.ObjectInfo) []reconcile.Object {
|
|
objects := make([]reconcile.Object, len(scanned))
|
|
for i, info := range scanned {
|
|
objects[i] = reconcile.Object{Key: info.Key, Size: info.Size, LastModified: info.LastModified}
|
|
}
|
|
return objects
|
|
}
|
|
|
|
func getenv(key, def string) string {
|
|
if value := os.Getenv(key); value != "" {
|
|
return value
|
|
}
|
|
return def
|
|
}
|