forked from mtusi2026/mtusi2026lab3
500 lines
16 KiB
Go
500 lines
16 KiB
Go
// SQL experiment adapter. Mutating modes create only fresh lab2_test_* schemas.
|
|
package main
|
|
|
|
import (
|
|
"context"
|
|
"crypto/sha256"
|
|
"encoding/hex"
|
|
"encoding/json"
|
|
"errors"
|
|
"flag"
|
|
"fmt"
|
|
repository2 "highload/internal/repository"
|
|
"highload/internal/repository/postgres"
|
|
"io"
|
|
"log/slog"
|
|
"os"
|
|
"os/signal"
|
|
"runtime"
|
|
"sort"
|
|
"strings"
|
|
"sync"
|
|
"syscall"
|
|
"time"
|
|
|
|
"highload/db/migrations"
|
|
"highload/internal/database/sqlc"
|
|
"highload/internal/model"
|
|
|
|
"github.com/google/uuid"
|
|
"github.com/jackc/pgx/v5"
|
|
"github.com/jackc/pgx/v5/pgxpool"
|
|
"github.com/jackc/pgx/v5/stdlib"
|
|
"github.com/pressly/goose/v3"
|
|
)
|
|
|
|
type recorder struct {
|
|
mu sync.Mutex
|
|
active bool
|
|
SQL string
|
|
args []any
|
|
}
|
|
|
|
func (r *recorder) TraceQueryStart(ctx context.Context, _ *pgx.Conn, data pgx.TraceQueryStartData) context.Context {
|
|
r.mu.Lock()
|
|
defer r.mu.Unlock()
|
|
if r.active {
|
|
r.SQL, r.args = data.SQL, append([]any(nil), data.Args...)
|
|
}
|
|
return ctx
|
|
}
|
|
func (*recorder) TraceQueryEnd(context.Context, *pgx.Conn, pgx.TraceQueryEndData) {}
|
|
|
|
type fixture struct {
|
|
admin *pgx.Conn
|
|
pool *pgxpool.Pool
|
|
schema string
|
|
owns bool
|
|
user string
|
|
chats []string
|
|
record recorder
|
|
researchBudget time.Duration
|
|
}
|
|
|
|
func (f *fixture) cleanup() error {
|
|
if f.pool != nil {
|
|
f.pool.Close()
|
|
}
|
|
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
|
defer cancel()
|
|
var cleanupErr error
|
|
if f.admin != nil {
|
|
if f.owns {
|
|
_, cleanupErr = f.admin.Exec(ctx, "DROP SCHEMA "+pgx.Identifier{f.schema}.Sanitize()+" CASCADE")
|
|
}
|
|
_ = f.admin.Close(ctx)
|
|
}
|
|
return cleanupErr
|
|
}
|
|
|
|
func createFixture(ctx context.Context, url, schema string, rows int) (_ *fixture, err error) {
|
|
f := &fixture{schema: schema}
|
|
defer func() {
|
|
if err != nil {
|
|
f.cleanup()
|
|
}
|
|
}()
|
|
f.admin, err = pgx.Connect(ctx, url)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
_, err = f.admin.Exec(ctx, "CREATE SCHEMA "+pgx.Identifier{schema}.Sanitize())
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
f.owns = true // Never drop a schema unless this invocation created it.
|
|
cfg, err := pgx.ParseConfig(url)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
cfg.RuntimeParams["search_path"] = schema
|
|
cfg.RuntimeParams["timezone"] = "UTC"
|
|
cfg.ConnectTimeout = 5 * time.Second
|
|
db := stdlib.OpenDB(*cfg)
|
|
provider, err := goose.NewProvider(goose.DialectPostgres, db, migrations.Files,
|
|
goose.WithVerbose(false), goose.WithDisableGlobalRegistry(true),
|
|
goose.WithSlog(slog.New(slog.NewJSONHandler(io.Discard, nil))))
|
|
if err == nil {
|
|
_, err = provider.Up(ctx)
|
|
}
|
|
db.Close()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
poolCfg, err := pgxpool.ParseConfig(url)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
poolCfg.MaxConns = 2
|
|
poolCfg.ConnConfig = cfg
|
|
poolCfg.ConnConfig.Tracer = &f.record
|
|
f.pool, err = pgxpool.NewWithConfig(ctx, poolCfg)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
f.user = uuid.NewString()
|
|
_, err = f.pool.Exec(ctx, "INSERT INTO users(id,username,display_name,password_hash) VALUES($1,$2,$3,$4)",
|
|
f.user, "sql_fixture", "Synthetic SQL fixture", "not-a-login-credential")
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
for i := 0; i < 5; i++ {
|
|
id := uuid.NewString()
|
|
f.chats = append(f.chats, id)
|
|
if _, err = f.pool.Exec(ctx, "INSERT INTO chats(id,title) VALUES($1,$2)", id, fmt.Sprintf("SQL chat %d", i)); err != nil {
|
|
return nil, err
|
|
}
|
|
if _, err = f.pool.Exec(ctx, "INSERT INTO chat_members(chat_id,user_id) VALUES($1,$2)", id, f.user); err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
_, err = f.pool.Exec(ctx, `INSERT INTO messages(chat_id,sender_id,text,created_at)
|
|
SELECT CASE WHEN n%100<70 THEN $1::uuid WHEN n%100<90 THEN $2::uuid
|
|
WHEN n%100<96 THEN $3::uuid WHEN n%100<99 THEN $4::uuid ELSE $5::uuid END,
|
|
$6::uuid, 'sql-fixture-'||n, timestamptz '2026-01-01 00:00:00+00'+n*interval '1 millisecond'
|
|
FROM generate_series(1,$7::integer) n`, f.chats[0], f.chats[1], f.chats[2], f.chats[3], f.chats[4], f.user, rows)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
_, err = f.pool.Exec(ctx, "ANALYZE messages")
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return f, nil
|
|
}
|
|
|
|
func (f *fixture) position(ctx context.Context) (*repository2.MessagePosition, error) {
|
|
var p repository2.MessagePosition
|
|
err := f.pool.QueryRow(ctx, `SELECT id::text,created_at FROM messages WHERE chat_id=$1
|
|
ORDER BY created_at,id OFFSET (SELECT count(*)*8/10 FROM messages WHERE chat_id=$1) LIMIT 1`, f.chats[0]).Scan(&p.ID, &p.CreatedAt)
|
|
// OFFSET is used only to select a fixture cursor, never in the measured query.
|
|
return &p, err
|
|
}
|
|
|
|
func (f *fixture) read(ctx context.Context, after *repository2.MessagePosition) (time.Duration, error) {
|
|
return f.readChat(ctx, f.chats[0], after)
|
|
}
|
|
|
|
func (f *fixture) readChat(ctx context.Context, chatID string, after *repository2.MessagePosition) (time.Duration, error) {
|
|
repo := postgres.NewMessageRepository(sqlc.New(f.pool), f.pool)
|
|
f.record.mu.Lock()
|
|
f.record.active = true
|
|
f.record.mu.Unlock()
|
|
started := time.Now()
|
|
var items []*model.Message
|
|
var err error
|
|
if f.researchBudget > 0 {
|
|
items, err = f.researchRead(ctx, chatID, after)
|
|
} else {
|
|
items, err = repo.GetMessagesByChatID(ctx, chatID, 51, after)
|
|
}
|
|
elapsed := time.Since(started)
|
|
f.record.mu.Lock()
|
|
f.record.active = false
|
|
f.record.mu.Unlock()
|
|
if err == nil && len(items) != 51 {
|
|
err = fmt.Errorf("fixture history returned %d rows instead of 51", len(items))
|
|
}
|
|
for i := 1; err == nil && i < len(items); i++ {
|
|
if items[i].CreatedAt.Before(items[i-1].CreatedAt) {
|
|
err = errors.New("history out of order")
|
|
}
|
|
}
|
|
return elapsed, err
|
|
}
|
|
|
|
func maxVisited(node map[string]any) float64 {
|
|
number := func(key string) float64 { value, _ := node[key].(float64); return value }
|
|
visited := (number("Actual Rows") + number("Rows Removed by Filter") + number("Rows Removed by Index Recheck")) * number("Actual Loops")
|
|
children, _ := node["Plans"].([]any)
|
|
for _, child := range children {
|
|
if child, ok := child.(map[string]any); ok {
|
|
visited = max(visited, maxVisited(child))
|
|
}
|
|
}
|
|
return visited
|
|
}
|
|
|
|
func (f *fixture) plan(ctx context.Context, after *repository2.MessagePosition) (map[string]any, error) {
|
|
return f.planChat(ctx, f.chats[0], after)
|
|
}
|
|
|
|
func (f *fixture) planChat(ctx context.Context, chatID string, after *repository2.MessagePosition) (map[string]any, error) {
|
|
if _, err := f.readChat(ctx, chatID, after); err != nil {
|
|
return nil, err
|
|
}
|
|
f.record.mu.Lock()
|
|
query, args := f.record.SQL, append([]any(nil), f.record.args...)
|
|
f.record.mu.Unlock()
|
|
if !strings.Contains(query, "GetMessagesByChatID") {
|
|
return nil, errors.New("adapt query recorder to repository SQL")
|
|
}
|
|
var raw []byte
|
|
err := f.pool.QueryRow(ctx, "EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON) "+query, args...).Scan(&raw)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
var plan []map[string]any
|
|
if err = json.Unmarshal(raw, &plan); err != nil || len(plan) != 1 {
|
|
return nil, errors.New("invalid EXPLAIN output")
|
|
}
|
|
node, ok := plan[0]["Plan"].(map[string]any)
|
|
if !ok {
|
|
return nil, errors.New("missing EXPLAIN plan")
|
|
}
|
|
return map[string]any{"explain": plan[0], "max_visited_rows": maxVisited(node)}, nil
|
|
}
|
|
|
|
func latency(values []time.Duration) map[string]float64 {
|
|
sort.Slice(values, func(i, j int) bool { return values[i] < values[j] })
|
|
result := map[string]float64{}
|
|
for _, p := range []int{50, 95, 99} {
|
|
i := (len(values)*p+99)/100 - 1
|
|
result[fmt.Sprintf("p%d", p)] = float64(values[i].Microseconds()) / 1000
|
|
}
|
|
return result
|
|
}
|
|
|
|
func (f *fixture) compare(ctx context.Context, repetitions int) ([]map[string]any, error) {
|
|
position, err := f.position(ctx)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
// Only this fresh fixture schema is changed; application migrations stay intact.
|
|
if _, err = f.pool.Exec(ctx, "DROP INDEX idx_messages_chat_created_id"); err != nil {
|
|
return nil, err
|
|
}
|
|
var results []map[string]any
|
|
for _, columns := range []string{"chat_id", "chat_id,created_at,id"} {
|
|
if _, err = f.pool.Exec(ctx, "CREATE INDEX lab2_history_variant ON messages("+columns+")"); err != nil {
|
|
return nil, err
|
|
}
|
|
if _, err = f.pool.Exec(ctx, "ANALYZE messages"); err != nil {
|
|
return nil, err
|
|
}
|
|
for i := 0; i < 3; i++ {
|
|
if _, err = f.read(ctx, position); err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
var reads, writes []time.Duration
|
|
for i := 0; i < repetitions; i++ {
|
|
duration, err := f.read(ctx, position)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
reads = append(reads, duration)
|
|
prefix := "write-fixture-" + uuid.NewString() + "-"
|
|
started := time.Now()
|
|
tx, err := f.pool.Begin(ctx)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
_, err = tx.Exec(ctx, `INSERT INTO messages(chat_id,sender_id,text)
|
|
SELECT $1::uuid,$2::uuid,$3::text||n FROM generate_series(1,200) n`, f.chats[0], f.user, prefix)
|
|
if err != nil {
|
|
_ = tx.Rollback(ctx)
|
|
return nil, err
|
|
}
|
|
if err = tx.Commit(ctx); err != nil {
|
|
return nil, err
|
|
}
|
|
writes = append(writes, time.Since(started))
|
|
if _, err = f.pool.Exec(ctx, "DELETE FROM messages WHERE text LIKE $1", prefix+"%"); err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
first, err := f.plan(ctx, nil)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
deep, err := f.plan(ctx, position)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
var size int64
|
|
if err = f.pool.QueryRow(ctx, "SELECT pg_relation_size('lab2_history_variant')").Scan(&size); err != nil {
|
|
return nil, err
|
|
}
|
|
results = append(results, map[string]any{"columns": columns, "read_ms": latency(reads), "insert_200_with_commit_ms": latency(writes), "index_bytes": size, "first_page": first, "deep_page": deep})
|
|
if _, err = f.pool.Exec(ctx, "DROP INDEX lab2_history_variant"); err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
return results, nil
|
|
}
|
|
|
|
func snapshot(ctx context.Context, url, schema string) (map[string]any, error) {
|
|
cfg, err := pgx.ParseConfig(url)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
cfg.RuntimeParams["timezone"] = "UTC"
|
|
cfg.RuntimeParams["datestyle"] = "ISO, YMD"
|
|
cfg.ConnectTimeout = 5 * time.Second
|
|
conn, err := pgx.ConnectConfig(ctx, cfg)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer conn.Close(context.Background())
|
|
tx, err := conn.BeginTx(ctx, pgx.TxOptions{IsoLevel: pgx.RepeatableRead, AccessMode: pgx.ReadOnly})
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer tx.Rollback(context.Background())
|
|
identity := sha256.Sum256([]byte(fmt.Sprintf("%s:%d/%s", cfg.Host, cfg.Port, cfg.Database)))
|
|
tables := map[string]any{}
|
|
missing := 0
|
|
for _, table := range []struct{ name, order string }{{"users", "id"}, {"chats", "id"}, {"chat_members", "chat_id,user_id"}, {"messages", "id"}} {
|
|
qualified := pgx.Identifier{schema, table.name}.Sanitize()
|
|
var exists bool
|
|
if err = tx.QueryRow(ctx, "SELECT to_regclass($1) IS NOT NULL", qualified).Scan(&exists); err != nil {
|
|
return nil, err
|
|
}
|
|
if !exists {
|
|
missing++
|
|
continue
|
|
}
|
|
rows, err := tx.Query(ctx, "SELECT row_to_json(t)::text FROM "+qualified+" t ORDER BY "+table.order)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
digest := sha256.New()
|
|
count := 0
|
|
for rows.Next() {
|
|
var row string
|
|
if err = rows.Scan(&row); err != nil {
|
|
rows.Close()
|
|
return nil, err
|
|
}
|
|
_, _ = io.WriteString(digest, row+"\n")
|
|
count++
|
|
}
|
|
err = rows.Err()
|
|
rows.Close()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
tables[table.name] = map[string]any{"rows": count, "sha256": hex.EncodeToString(digest.Sum(nil))}
|
|
}
|
|
if missing != 0 && missing != 4 {
|
|
return nil, errors.New("snapshot schema contains only part of the application tables")
|
|
}
|
|
return map[string]any{"identity_sha256": hex.EncodeToString(identity[:]), "empty": missing == 4, "tables": tables}, nil
|
|
}
|
|
|
|
func execute(ctx context.Context, mode, url, schema string, rows, budget, repetitions int) (result map[string]any, err error) {
|
|
if mode == "snapshot" {
|
|
return snapshot(ctx, url, schema)
|
|
}
|
|
if rows < 10000 || ((mode == "compare" || mode == "seed") && rows < 100000) || repetitions < 2 || budget < 1 {
|
|
return nil, errors.New("invalid experiment limits")
|
|
}
|
|
if schema == "public" || mode != "seed" {
|
|
schema = "lab2_test_" + strings.ReplaceAll(uuid.NewString(), "-", "")
|
|
}
|
|
if !strings.HasPrefix(schema, "lab2_test_") {
|
|
return nil, errors.New("fixture schema must start with lab2_test_")
|
|
}
|
|
f, err := createFixture(ctx, url, schema, rows)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer func() {
|
|
if cleanupErr := f.cleanup(); err == nil {
|
|
err = cleanupErr
|
|
}
|
|
}()
|
|
result = map[string]any{"mode": mode, "rows": rows, "schema": schema, "chat_distribution_percent": []int{70, 20, 6, 3, 1}}
|
|
var actualRows int
|
|
var postgresVersion string
|
|
if err = f.pool.QueryRow(ctx, "SELECT count(*) FROM messages").Scan(&actualRows); err != nil || actualRows != rows {
|
|
return nil, errors.New("fixture row count differs from the requested dataset")
|
|
}
|
|
if err = f.pool.QueryRow(ctx, "SHOW server_version").Scan(&postgresVersion); err != nil {
|
|
return nil, err
|
|
}
|
|
distribution, queryErr := f.pool.Query(ctx, "SELECT count(*) FROM messages GROUP BY chat_id ORDER BY count(*) DESC")
|
|
if queryErr != nil {
|
|
return nil, queryErr
|
|
}
|
|
var chatCounts []int
|
|
for distribution.Next() {
|
|
var count int
|
|
if err = distribution.Scan(&count); err != nil {
|
|
distribution.Close()
|
|
return nil, err
|
|
}
|
|
chatCounts = append(chatCounts, count)
|
|
}
|
|
err = distribution.Err()
|
|
distribution.Close()
|
|
if err != nil || len(chatCounts) != 5 || chatCounts[0] <= chatCounts[4] {
|
|
return nil, errors.New("fixture does not demonstrate skewed chat sizes")
|
|
}
|
|
result["actual_rows"], result["chat_message_counts"] = actualRows, chatCounts
|
|
result["postgres_version"], result["go_version"] = postgresVersion, runtime.Version()
|
|
if mode == "seed" {
|
|
f.owns = false
|
|
result["kept_fixture"] = true
|
|
return result, nil
|
|
}
|
|
if mode == "compare" {
|
|
variants, err := f.compare(ctx, repetitions)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
result["variants"] = variants
|
|
result["limitations"] = "Warm single-host prototype; write cleanup can leave bloat. Repeat under fixed resources and review plans, cold/generic plans and loss boundaries."
|
|
return result, nil
|
|
}
|
|
if mode != "plan" {
|
|
return nil, errors.New("unknown mode")
|
|
}
|
|
position, err := f.position(ctx)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
first, err := f.plan(ctx, nil)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
deep, err := f.plan(ctx, position)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if first["max_visited_rows"].(float64) > float64(budget) || deep["max_visited_rows"].(float64) > float64(budget) {
|
|
return nil, errors.New("history exceeds chosen scan budget")
|
|
}
|
|
result["first_page"], result["deep_page"], result["scan_budget_rows"] = first, deep, budget
|
|
return result, nil
|
|
}
|
|
|
|
func main() {
|
|
mode := flag.String("mode", "snapshot", "snapshot, plan, compare or seed")
|
|
schema := flag.String("schema", "public", "snapshot schema or fresh lab2_test_* seed schema")
|
|
rows := flag.Int("rows", 10000, "fixture message count")
|
|
budget := flag.Int("scan-budget", 1000, "engineering threshold, not a teacher-mandated constant")
|
|
repetitions := flag.Int("repetitions", 5, "measurement samples per variant")
|
|
series := flag.Int("series", 4, "independent AB/BA research rounds (even, >= 4)")
|
|
writeBatches := flag.Int("write-batches", 20, "committed batches of 200 rows per research trial")
|
|
timeout := flag.Duration("timeout", 3*time.Minute, "total SQL probe execution budget")
|
|
flag.Parse()
|
|
url := os.Getenv("LAB2_SQL_DATABASE_URL")
|
|
if url == "" {
|
|
fmt.Fprintln(os.Stderr, "set LAB2_SQL_DATABASE_URL for a dedicated test database")
|
|
os.Exit(2)
|
|
}
|
|
if *mode != "snapshot" && *mode != "plan" && *mode != "compare" && *mode != "seed" && *mode != "research" {
|
|
os.Exit(2)
|
|
}
|
|
signals, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
|
defer stop()
|
|
ctx, cancel := context.WithTimeout(signals, *timeout)
|
|
defer cancel()
|
|
var result map[string]any
|
|
var err error
|
|
if *mode == "research" {
|
|
result, err = research(ctx, url, *rows, *repetitions, *series, *writeBatches)
|
|
} else {
|
|
result, err = execute(ctx, *mode, url, *schema, *rows, *budget, *repetitions)
|
|
}
|
|
if err != nil {
|
|
fmt.Fprintln(os.Stderr, "SQL probe failed; private error details suppressed")
|
|
os.Exit(1)
|
|
}
|
|
if err = json.NewEncoder(os.Stdout).Encode(result); err != nil {
|
|
os.Exit(1)
|
|
}
|
|
}
|