Files
mtusi2026lab5/tools/reconcilebench/main.go
T

396 lines
12 KiB
Go

// Command reconcilebench measures reconciliation against disposable real
// PostgreSQL and S3 services, independently of the compose stand.
package main
import (
"bytes"
"context"
"crypto/sha256"
"encoding/json"
"errors"
"flag"
"fmt"
"io"
"os"
"os/signal"
"path/filepath"
"runtime"
"strings"
"syscall"
"time"
"github.com/google/uuid"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgxpool"
"messenger/internal/attachment"
"messenger/internal/testenv"
)
func main() {
o, err := parseOptions(os.Args[1:], os.Stderr)
if errors.Is(err, flag.ErrHelp) {
return
}
if err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(2)
}
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
ctx, cancel := context.WithTimeout(ctx, 30*time.Minute)
var records bytes.Buffer
var resultOut io.Writer = os.Stdout
if o.output != "" {
resultOut = io.MultiWriter(os.Stdout, &records)
}
err = exercise(ctx, o, resultOut, os.Stderr)
if o.output != "" {
if saveErr := writeResults(o.output, records.Bytes()); saveErr != nil {
err = errors.Join(err, fmt.Errorf("save results: %w", saveErr))
}
}
cancel()
stop()
if err != nil {
fmt.Fprintln(os.Stderr, "reconcilebench:", err)
os.Exit(1)
}
}
func writeResults(path string, records []byte) error {
decoder := json.NewDecoder(bytes.NewReader(records))
rows := make([]json.RawMessage, 0)
for {
var row json.RawMessage
err := decoder.Decode(&row)
if errors.Is(err, io.EOF) {
break
}
if err != nil {
return err
}
rows = append(rows, row)
}
raw, err := json.MarshalIndent(rows, "", " ")
if err != nil {
return err
}
if err = os.MkdirAll(filepath.Dir(path), 0755); err != nil {
return err
}
return os.WriteFile(path, append(raw, '\n'), 0644)
}
type measurement struct {
Phase string `json:"phase"`
Mode string `json:"mode"`
TargetFiles int `json:"target_files"`
GoHeapPeakBytes uint64 `json:"go_heap_peak_bytes"`
Stats attachment.Stats `json:"stats"`
Findings []attachment.Finding `json:"findings,omitempty"`
}
// Heap samples cover this Go process during Run. They exclude service processes
// and are deliberately not described as process RSS or total deployment memory.
func measure(ctx context.Context, r *attachment.Reconciler, phase, mode string, n int, apply, deep bool, out io.Writer) (attachment.Stats, error) {
done := make(chan struct{})
peakCh := make(chan uint64, 1)
go func() {
var mem runtime.MemStats
runtime.ReadMemStats(&mem)
peak := mem.HeapAlloc
ticker := time.NewTicker(10 * time.Millisecond)
defer ticker.Stop()
for {
select {
case <-ticker.C:
runtime.ReadMemStats(&mem)
if mem.HeapAlloc > peak {
peak = mem.HeapAlloc
}
case <-done:
runtime.ReadMemStats(&mem)
if mem.HeapAlloc > peak {
peak = mem.HeapAlloc
}
peakCh <- peak
return
}
}
}()
m := measurement{Phase: phase, Mode: mode, TargetFiles: n}
options := attachment.ReconcileOptions{Apply: apply, Deep: deep}
stats, err := r.Run(ctx, options, func(f attachment.Finding) { m.Findings = append(m.Findings, f) })
close(done)
m.GoHeapPeakBytes = <-peakCh
m.Stats = stats
if writeErr := json.NewEncoder(out).Encode(m); writeErr != nil {
return stats, writeErr
}
return stats, err
}
func exercise(ctx context.Context, o options, out, progress io.Writer) error {
db, err := testenv.StartPostgres(ctx)
if err != nil {
return err
}
defer db.Close()
s3, err := testenv.StartS3(ctx)
if err != nil {
return err
}
defer s3.Close()
if err = json.NewEncoder(out).Encode(map[string]any{
"phase": "environment", "postgres_image": testenv.PostgresImage, "s3_image": testenv.S3Image,
"grace_ms": o.grace.Milliseconds(), "wait_ms": o.wait.Milliseconds(),
"memory_metric": "sampled Go process HeapAlloc bytes during Run; not RSS; excludes PostgreSQL and S3",
}); err != nil {
return err
}
f, err := seedFaults(ctx, db.Pool, s3.Store)
if err != nil {
return err
}
fmt.Fprintf(progress, "fault objects seeded; waiting %s for real S3 timestamps to age\n", o.wait)
if err = waitForAge(ctx, o.wait, progress); err != nil {
return err
}
if err = f.seedFresh(ctx); err != nil {
return err
}
r, err := attachment.NewReconciler(db.Pool, s3.Store, o.grace)
if err != nil {
return err
}
for _, run := range []struct {
mode string
apply bool
repairs, findings int64
}{{"dry", false, 0, 6}, {"apply", true, 5, 6}, {"repeat", true, 0, 1}} {
stats, err := measure(ctx, r, "fault", run.mode, 7, run.apply, true, out)
if err != nil {
return err
}
if stats.Repaired != run.repairs || stats.Findings != run.findings {
return fmt.Errorf("fault %s: wanted %d repairs/%d findings, got %+v", run.mode, run.repairs, run.findings, stats)
}
if err = f.assertState(ctx, run.apply); err != nil {
return fmt.Errorf("fault %s: %w", run.mode, err)
}
}
fmt.Fprintln(progress, "fault dry/apply/repeat passed; referenced messages, corrupt used blobs and fresh objects preserved")
// These resources were created above solely for this run; no compose
// endpoint or pre-existing database/bucket is ever accepted by this tool.
if _, err = db.Pool.Exec(ctx, `TRUNCATE message_attachments,attachments,files CASCADE`); err != nil {
return err
}
if err = emptyBucket(ctx, s3.Store); err != nil {
return err
}
seeded := 0
for _, n := range o.sizes {
for i := seeded; i < n; i++ {
if err = seedScaleFile(ctx, db.Pool, s3.Store, f.user, f.chat, i); err != nil {
return err
}
if (i+1)%500 == 0 {
fmt.Fprintf(progress, "seeded %d referenced real objects\n", i+1)
}
}
seeded = n
stats, err := measure(ctx, r, "cost", "dry", n, false, false, out)
if err != nil {
return err
}
if stats.ScannedFiles != int64(n) || stats.ScannedObjects != int64(n) || stats.Findings != 0 || stats.Repaired != 0 {
return fmt.Errorf("cost %d: inconsistent fixture %+v", n, stats)
}
}
return json.NewEncoder(out).Encode(map[string]any{"phase": "complete", "fault_checks": "passed", "sizes": o.sizes})
}
func waitForAge(ctx context.Context, d time.Duration, progress io.Writer) error {
start := time.Now()
timer := time.NewTimer(d)
defer timer.Stop()
ticker := time.NewTicker(30 * time.Second)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return ctx.Err()
case <-timer.C:
return nil
case <-ticker.C:
fmt.Fprintf(progress, "aging real S3 objects: %s / %s\n", time.Since(start).Round(time.Second), d)
}
}
}
type benchFile struct {
hash []byte
before, after string
}
type benchObject struct {
key string
after bool
}
type faultFixture struct {
db *pgxpool.Pool
blobs attachment.Blobs
user, chat uuid.UUID
files []benchFile
objects []benchObject
}
func seedFaults(ctx context.Context, db *pgxpool.Pool, blobs attachment.Blobs) (*faultFixture, error) {
f := &faultFixture{db: db, blobs: blobs, user: uuid.New(), chat: uuid.New()}
if _, err := db.Exec(ctx, `INSERT INTO users(id,username,display_name,password_hash,created_at) VALUES($1,$2,'bench','unused',now())`, f.user, "bench-"+f.user.String()); err != nil {
return nil, err
}
if _, err := db.Exec(ctx, `INSERT INTO chats(id,title,created_at) VALUES($1,'reconcilebench',now())`, f.chat); err != nil {
return nil, err
}
if err := f.put(ctx, "staging/old", []byte("stale staging"), false); err != nil {
return nil, err
}
orphan := sha256.Sum256([]byte("old orphan"))
if err := f.put(ctx, attachment.BlobKey(orphan[:]), []byte("old orphan"), false); err != nil {
return nil, err
}
for _, c := range []struct {
label, status, after string
used, present, corrupt bool
}{
{"pending", "pending", "", false, true, false},
{"unused", "ready", "", false, true, false},
{"used", "ready", "ready", true, true, false},
{"corrupt", "ready", "ready", true, true, true},
{"missing", "ready", "failed", true, false, false},
{"already_failed", "failed", "failed", true, false, false},
} {
if err := f.file(ctx, c.label, c.status, c.after, c.used, c.present, c.corrupt, false); err != nil {
return nil, err
}
}
return f, nil
}
func (f *faultFixture) put(ctx context.Context, key string, body []byte, keep bool) error {
if err := f.blobs.Put(ctx, key, bytes.NewReader(body), int64(len(body)), "image/png"); err != nil {
return err
}
f.objects = append(f.objects, benchObject{key: key, after: keep})
return nil
}
func (f *faultFixture) file(ctx context.Context, label, status, after string, used, present, corrupt, fresh bool) error {
body := []byte("fixture/" + label)
hash := sha256.Sum256(body)
created := time.Now().Add(-2 * time.Hour)
if fresh {
created = time.Now()
}
if _, err := f.db.Exec(ctx, `INSERT INTO files(hash,size_bytes,mime_type,status,created_at) VALUES($1,$2,'image/png',$3,$4)`, hash[:], len(body), status, created); err != nil {
return err
}
f.files = append(f.files, benchFile{hash: hash[:], before: status, after: after})
if used {
id, message := uuid.New(), uuid.New()
if _, err := f.db.Exec(ctx, `INSERT INTO attachments(id,chat_id,owner_id,filename,file_hash,created_at) VALUES($1,$2,$3,'fixture.png',$4,now())`, id, f.chat, f.user, hash[:]); err != nil {
return err
}
if _, err := f.db.Exec(ctx, `INSERT INTO messages(chat_id,id,sender_id,text,created_at) VALUES($1,$2,$3,'preserve',now())`, f.chat, message, f.user); err != nil {
return err
}
if _, err := f.db.Exec(ctx, `INSERT INTO message_attachments(chat_id,message_id,attachment_id,ordinal) VALUES($1,$2,$3,0)`, f.chat, message, id); err != nil {
return err
}
}
if present {
if corrupt {
body[0] ^= 1
}
return f.put(ctx, attachment.BlobKey(hash[:]), body, after != "")
}
return nil
}
func (f *faultFixture) seedFresh(ctx context.Context) error {
if err := f.put(ctx, "staging/fresh", []byte("fresh staging"), true); err != nil {
return err
}
orphan := sha256.Sum256([]byte("fresh orphan"))
if err := f.put(ctx, attachment.BlobKey(orphan[:]), []byte("fresh orphan"), true); err != nil {
return err
}
return f.file(ctx, "fresh pending", "pending", "pending", false, true, false, true)
}
func (f *faultFixture) assertState(ctx context.Context, applied bool) error {
for _, file := range f.files {
var status string
err := f.db.QueryRow(ctx, `SELECT status FROM files WHERE hash=$1`, file.hash).Scan(&status)
if err != nil && !errors.Is(err, pgx.ErrNoRows) {
return err
}
want := file.before
if applied {
want = file.after
}
if status != want {
return fmt.Errorf("file %s: status=%q, want %q", attachment.BlobKey(file.hash), status, want)
}
}
for _, obj := range f.objects {
_, err := f.blobs.Stat(ctx, obj.key)
want := !applied || obj.after
if err != nil && !errors.Is(err, attachment.ErrObjectMissing) {
return err
}
if (err == nil) != want {
return fmt.Errorf("object %s: present=%v want %v", obj.key, err == nil, want)
}
}
var attachments, links int
if err := f.db.QueryRow(ctx, `SELECT (SELECT count(*) FROM attachments),(SELECT count(*) FROM message_attachments)`).Scan(&attachments, &links); err != nil {
return err
}
if attachments != 4 || links != 4 {
return fmt.Errorf("referenced attachments/messages lost: %d/%d", attachments, links)
}
return nil
}
func emptyBucket(ctx context.Context, blobs attachment.Blobs) error {
for {
page, err := blobs.List(ctx, "", "", 500)
if err != nil {
return err
}
if len(page) == 0 {
return nil
}
for _, obj := range page {
if err := blobs.Delete(ctx, obj.Key); err != nil {
return err
}
}
}
}
func seedScaleFile(ctx context.Context, db *pgxpool.Pool, blobs attachment.Blobs, user, chat uuid.UUID, i int) error {
body := fmt.Sprintf("scale/%09d", i)
hash := sha256.Sum256([]byte(body))
if err := blobs.Put(ctx, attachment.BlobKey(hash[:]), strings.NewReader(body), int64(len(body)), "image/png"); err != nil {
return err
}
_, err := db.Exec(ctx, `WITH f AS (
INSERT INTO files(hash,size_bytes,mime_type,status,created_at)
VALUES($1,$2,'image/png','ready',now()-interval '2 hour') RETURNING hash)
INSERT INTO attachments(id,chat_id,owner_id,filename,file_hash,created_at)
SELECT $3,$4,$5,'scale.png',hash,now() FROM f`, hash[:], len(body), uuid.New(), chat, user)
return err
}