396 lines
12 KiB
Go
396 lines
12 KiB
Go
// Command reconcilebench measures reconciliation against disposable real
|
|
// PostgreSQL and S3 services, independently of the compose stand.
|
|
package main
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"crypto/sha256"
|
|
"encoding/json"
|
|
"errors"
|
|
"flag"
|
|
"fmt"
|
|
"io"
|
|
"os"
|
|
"os/signal"
|
|
"path/filepath"
|
|
"runtime"
|
|
"strings"
|
|
"syscall"
|
|
"time"
|
|
|
|
"github.com/google/uuid"
|
|
"github.com/jackc/pgx/v5"
|
|
"github.com/jackc/pgx/v5/pgxpool"
|
|
|
|
"messenger/internal/attachment"
|
|
"messenger/internal/testenv"
|
|
)
|
|
|
|
func main() {
|
|
o, err := parseOptions(os.Args[1:], os.Stderr)
|
|
if errors.Is(err, flag.ErrHelp) {
|
|
return
|
|
}
|
|
if err != nil {
|
|
fmt.Fprintln(os.Stderr, err)
|
|
os.Exit(2)
|
|
}
|
|
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
|
|
ctx, cancel := context.WithTimeout(ctx, 30*time.Minute)
|
|
var records bytes.Buffer
|
|
var resultOut io.Writer = os.Stdout
|
|
if o.output != "" {
|
|
resultOut = io.MultiWriter(os.Stdout, &records)
|
|
}
|
|
err = exercise(ctx, o, resultOut, os.Stderr)
|
|
if o.output != "" {
|
|
if saveErr := writeResults(o.output, records.Bytes()); saveErr != nil {
|
|
err = errors.Join(err, fmt.Errorf("save results: %w", saveErr))
|
|
}
|
|
}
|
|
cancel()
|
|
stop()
|
|
if err != nil {
|
|
fmt.Fprintln(os.Stderr, "reconcilebench:", err)
|
|
os.Exit(1)
|
|
}
|
|
}
|
|
|
|
func writeResults(path string, records []byte) error {
|
|
decoder := json.NewDecoder(bytes.NewReader(records))
|
|
rows := make([]json.RawMessage, 0)
|
|
for {
|
|
var row json.RawMessage
|
|
err := decoder.Decode(&row)
|
|
if errors.Is(err, io.EOF) {
|
|
break
|
|
}
|
|
if err != nil {
|
|
return err
|
|
}
|
|
rows = append(rows, row)
|
|
}
|
|
raw, err := json.MarshalIndent(rows, "", " ")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if err = os.MkdirAll(filepath.Dir(path), 0755); err != nil {
|
|
return err
|
|
}
|
|
return os.WriteFile(path, append(raw, '\n'), 0644)
|
|
}
|
|
|
|
type measurement struct {
|
|
Phase string `json:"phase"`
|
|
Mode string `json:"mode"`
|
|
TargetFiles int `json:"target_files"`
|
|
GoHeapPeakBytes uint64 `json:"go_heap_peak_bytes"`
|
|
Stats attachment.Stats `json:"stats"`
|
|
Findings []attachment.Finding `json:"findings,omitempty"`
|
|
}
|
|
|
|
// Heap samples cover this Go process during Run. They exclude service processes
|
|
// and are deliberately not described as process RSS or total deployment memory.
|
|
func measure(ctx context.Context, r *attachment.Reconciler, phase, mode string, n int, apply, deep bool, out io.Writer) (attachment.Stats, error) {
|
|
done := make(chan struct{})
|
|
peakCh := make(chan uint64, 1)
|
|
go func() {
|
|
var mem runtime.MemStats
|
|
runtime.ReadMemStats(&mem)
|
|
peak := mem.HeapAlloc
|
|
ticker := time.NewTicker(10 * time.Millisecond)
|
|
defer ticker.Stop()
|
|
for {
|
|
select {
|
|
case <-ticker.C:
|
|
runtime.ReadMemStats(&mem)
|
|
if mem.HeapAlloc > peak {
|
|
peak = mem.HeapAlloc
|
|
}
|
|
case <-done:
|
|
runtime.ReadMemStats(&mem)
|
|
if mem.HeapAlloc > peak {
|
|
peak = mem.HeapAlloc
|
|
}
|
|
peakCh <- peak
|
|
return
|
|
}
|
|
}
|
|
}()
|
|
m := measurement{Phase: phase, Mode: mode, TargetFiles: n}
|
|
options := attachment.ReconcileOptions{Apply: apply, Deep: deep}
|
|
stats, err := r.Run(ctx, options, func(f attachment.Finding) { m.Findings = append(m.Findings, f) })
|
|
close(done)
|
|
m.GoHeapPeakBytes = <-peakCh
|
|
m.Stats = stats
|
|
if writeErr := json.NewEncoder(out).Encode(m); writeErr != nil {
|
|
return stats, writeErr
|
|
}
|
|
return stats, err
|
|
}
|
|
|
|
func exercise(ctx context.Context, o options, out, progress io.Writer) error {
|
|
db, err := testenv.StartPostgres(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer db.Close()
|
|
s3, err := testenv.StartS3(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer s3.Close()
|
|
if err = json.NewEncoder(out).Encode(map[string]any{
|
|
"phase": "environment", "postgres_image": testenv.PostgresImage, "s3_image": testenv.S3Image,
|
|
"grace_ms": o.grace.Milliseconds(), "wait_ms": o.wait.Milliseconds(),
|
|
"memory_metric": "sampled Go process HeapAlloc bytes during Run; not RSS; excludes PostgreSQL and S3",
|
|
}); err != nil {
|
|
return err
|
|
}
|
|
f, err := seedFaults(ctx, db.Pool, s3.Store)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
fmt.Fprintf(progress, "fault objects seeded; waiting %s for real S3 timestamps to age\n", o.wait)
|
|
if err = waitForAge(ctx, o.wait, progress); err != nil {
|
|
return err
|
|
}
|
|
if err = f.seedFresh(ctx); err != nil {
|
|
return err
|
|
}
|
|
r, err := attachment.NewReconciler(db.Pool, s3.Store, o.grace)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
for _, run := range []struct {
|
|
mode string
|
|
apply bool
|
|
repairs, findings int64
|
|
}{{"dry", false, 0, 6}, {"apply", true, 5, 6}, {"repeat", true, 0, 1}} {
|
|
stats, err := measure(ctx, r, "fault", run.mode, 7, run.apply, true, out)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if stats.Repaired != run.repairs || stats.Findings != run.findings {
|
|
return fmt.Errorf("fault %s: wanted %d repairs/%d findings, got %+v", run.mode, run.repairs, run.findings, stats)
|
|
}
|
|
if err = f.assertState(ctx, run.apply); err != nil {
|
|
return fmt.Errorf("fault %s: %w", run.mode, err)
|
|
}
|
|
}
|
|
fmt.Fprintln(progress, "fault dry/apply/repeat passed; referenced messages, corrupt used blobs and fresh objects preserved")
|
|
// These resources were created above solely for this run; no compose
|
|
// endpoint or pre-existing database/bucket is ever accepted by this tool.
|
|
if _, err = db.Pool.Exec(ctx, `TRUNCATE message_attachments,attachments,files CASCADE`); err != nil {
|
|
return err
|
|
}
|
|
if err = emptyBucket(ctx, s3.Store); err != nil {
|
|
return err
|
|
}
|
|
seeded := 0
|
|
for _, n := range o.sizes {
|
|
for i := seeded; i < n; i++ {
|
|
if err = seedScaleFile(ctx, db.Pool, s3.Store, f.user, f.chat, i); err != nil {
|
|
return err
|
|
}
|
|
if (i+1)%500 == 0 {
|
|
fmt.Fprintf(progress, "seeded %d referenced real objects\n", i+1)
|
|
}
|
|
}
|
|
seeded = n
|
|
stats, err := measure(ctx, r, "cost", "dry", n, false, false, out)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if stats.ScannedFiles != int64(n) || stats.ScannedObjects != int64(n) || stats.Findings != 0 || stats.Repaired != 0 {
|
|
return fmt.Errorf("cost %d: inconsistent fixture %+v", n, stats)
|
|
}
|
|
}
|
|
return json.NewEncoder(out).Encode(map[string]any{"phase": "complete", "fault_checks": "passed", "sizes": o.sizes})
|
|
}
|
|
|
|
func waitForAge(ctx context.Context, d time.Duration, progress io.Writer) error {
|
|
start := time.Now()
|
|
timer := time.NewTimer(d)
|
|
defer timer.Stop()
|
|
ticker := time.NewTicker(30 * time.Second)
|
|
defer ticker.Stop()
|
|
for {
|
|
select {
|
|
case <-ctx.Done():
|
|
return ctx.Err()
|
|
case <-timer.C:
|
|
return nil
|
|
case <-ticker.C:
|
|
fmt.Fprintf(progress, "aging real S3 objects: %s / %s\n", time.Since(start).Round(time.Second), d)
|
|
}
|
|
}
|
|
}
|
|
|
|
type benchFile struct {
|
|
hash []byte
|
|
before, after string
|
|
}
|
|
type benchObject struct {
|
|
key string
|
|
after bool
|
|
}
|
|
type faultFixture struct {
|
|
db *pgxpool.Pool
|
|
blobs attachment.Blobs
|
|
user, chat uuid.UUID
|
|
files []benchFile
|
|
objects []benchObject
|
|
}
|
|
|
|
func seedFaults(ctx context.Context, db *pgxpool.Pool, blobs attachment.Blobs) (*faultFixture, error) {
|
|
f := &faultFixture{db: db, blobs: blobs, user: uuid.New(), chat: uuid.New()}
|
|
if _, err := db.Exec(ctx, `INSERT INTO users(id,username,display_name,password_hash,created_at) VALUES($1,$2,'bench','unused',now())`, f.user, "bench-"+f.user.String()); err != nil {
|
|
return nil, err
|
|
}
|
|
if _, err := db.Exec(ctx, `INSERT INTO chats(id,title,created_at) VALUES($1,'reconcilebench',now())`, f.chat); err != nil {
|
|
return nil, err
|
|
}
|
|
if err := f.put(ctx, "staging/old", []byte("stale staging"), false); err != nil {
|
|
return nil, err
|
|
}
|
|
orphan := sha256.Sum256([]byte("old orphan"))
|
|
if err := f.put(ctx, attachment.BlobKey(orphan[:]), []byte("old orphan"), false); err != nil {
|
|
return nil, err
|
|
}
|
|
for _, c := range []struct {
|
|
label, status, after string
|
|
used, present, corrupt bool
|
|
}{
|
|
{"pending", "pending", "", false, true, false},
|
|
{"unused", "ready", "", false, true, false},
|
|
{"used", "ready", "ready", true, true, false},
|
|
{"corrupt", "ready", "ready", true, true, true},
|
|
{"missing", "ready", "failed", true, false, false},
|
|
{"already_failed", "failed", "failed", true, false, false},
|
|
} {
|
|
if err := f.file(ctx, c.label, c.status, c.after, c.used, c.present, c.corrupt, false); err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
return f, nil
|
|
}
|
|
|
|
func (f *faultFixture) put(ctx context.Context, key string, body []byte, keep bool) error {
|
|
if err := f.blobs.Put(ctx, key, bytes.NewReader(body), int64(len(body)), "image/png"); err != nil {
|
|
return err
|
|
}
|
|
f.objects = append(f.objects, benchObject{key: key, after: keep})
|
|
return nil
|
|
}
|
|
|
|
func (f *faultFixture) file(ctx context.Context, label, status, after string, used, present, corrupt, fresh bool) error {
|
|
body := []byte("fixture/" + label)
|
|
hash := sha256.Sum256(body)
|
|
created := time.Now().Add(-2 * time.Hour)
|
|
if fresh {
|
|
created = time.Now()
|
|
}
|
|
if _, err := f.db.Exec(ctx, `INSERT INTO files(hash,size_bytes,mime_type,status,created_at) VALUES($1,$2,'image/png',$3,$4)`, hash[:], len(body), status, created); err != nil {
|
|
return err
|
|
}
|
|
f.files = append(f.files, benchFile{hash: hash[:], before: status, after: after})
|
|
if used {
|
|
id, message := uuid.New(), uuid.New()
|
|
if _, err := f.db.Exec(ctx, `INSERT INTO attachments(id,chat_id,owner_id,filename,file_hash,created_at) VALUES($1,$2,$3,'fixture.png',$4,now())`, id, f.chat, f.user, hash[:]); err != nil {
|
|
return err
|
|
}
|
|
if _, err := f.db.Exec(ctx, `INSERT INTO messages(chat_id,id,sender_id,text,created_at) VALUES($1,$2,$3,'preserve',now())`, f.chat, message, f.user); err != nil {
|
|
return err
|
|
}
|
|
if _, err := f.db.Exec(ctx, `INSERT INTO message_attachments(chat_id,message_id,attachment_id,ordinal) VALUES($1,$2,$3,0)`, f.chat, message, id); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
if present {
|
|
if corrupt {
|
|
body[0] ^= 1
|
|
}
|
|
return f.put(ctx, attachment.BlobKey(hash[:]), body, after != "")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (f *faultFixture) seedFresh(ctx context.Context) error {
|
|
if err := f.put(ctx, "staging/fresh", []byte("fresh staging"), true); err != nil {
|
|
return err
|
|
}
|
|
orphan := sha256.Sum256([]byte("fresh orphan"))
|
|
if err := f.put(ctx, attachment.BlobKey(orphan[:]), []byte("fresh orphan"), true); err != nil {
|
|
return err
|
|
}
|
|
return f.file(ctx, "fresh pending", "pending", "pending", false, true, false, true)
|
|
}
|
|
|
|
func (f *faultFixture) assertState(ctx context.Context, applied bool) error {
|
|
for _, file := range f.files {
|
|
var status string
|
|
err := f.db.QueryRow(ctx, `SELECT status FROM files WHERE hash=$1`, file.hash).Scan(&status)
|
|
if err != nil && !errors.Is(err, pgx.ErrNoRows) {
|
|
return err
|
|
}
|
|
want := file.before
|
|
if applied {
|
|
want = file.after
|
|
}
|
|
if status != want {
|
|
return fmt.Errorf("file %s: status=%q, want %q", attachment.BlobKey(file.hash), status, want)
|
|
}
|
|
}
|
|
for _, obj := range f.objects {
|
|
_, err := f.blobs.Stat(ctx, obj.key)
|
|
want := !applied || obj.after
|
|
if err != nil && !errors.Is(err, attachment.ErrObjectMissing) {
|
|
return err
|
|
}
|
|
if (err == nil) != want {
|
|
return fmt.Errorf("object %s: present=%v want %v", obj.key, err == nil, want)
|
|
}
|
|
}
|
|
var attachments, links int
|
|
if err := f.db.QueryRow(ctx, `SELECT (SELECT count(*) FROM attachments),(SELECT count(*) FROM message_attachments)`).Scan(&attachments, &links); err != nil {
|
|
return err
|
|
}
|
|
if attachments != 4 || links != 4 {
|
|
return fmt.Errorf("referenced attachments/messages lost: %d/%d", attachments, links)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func emptyBucket(ctx context.Context, blobs attachment.Blobs) error {
|
|
for {
|
|
page, err := blobs.List(ctx, "", "", 500)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if len(page) == 0 {
|
|
return nil
|
|
}
|
|
for _, obj := range page {
|
|
if err := blobs.Delete(ctx, obj.Key); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
func seedScaleFile(ctx context.Context, db *pgxpool.Pool, blobs attachment.Blobs, user, chat uuid.UUID, i int) error {
|
|
body := fmt.Sprintf("scale/%09d", i)
|
|
hash := sha256.Sum256([]byte(body))
|
|
if err := blobs.Put(ctx, attachment.BlobKey(hash[:]), strings.NewReader(body), int64(len(body)), "image/png"); err != nil {
|
|
return err
|
|
}
|
|
_, err := db.Exec(ctx, `WITH f AS (
|
|
INSERT INTO files(hash,size_bytes,mime_type,status,created_at)
|
|
VALUES($1,$2,'image/png','ready',now()-interval '2 hour') RETURNING hash)
|
|
INSERT INTO attachments(id,chat_id,owner_id,filename,file_hash,created_at)
|
|
SELECT $3,$4,$5,'scale.png',hash,now() FROM f`, hash[:], len(body), uuid.New(), chat, user)
|
|
return err
|
|
}
|