Files
mtusi2026lab2/student-tests/lab2/sqlprobe/main.go
T
OlegRood 9f0bb97b09
Lab2 application / application (pull_request) Canceled after 0s
ЛР 2: завершить трек 2 на оценку 5
2026-10-28 15:41:38 +03:00

500 lines
16 KiB
Go

// SQL experiment adapter. Mutating modes create only fresh lab2_test_* schemas.
package main
import (
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"flag"
"fmt"
repository2 "highload/internal/repository"
"highload/internal/repository/postgres"
"io"
"log/slog"
"os"
"os/signal"
"runtime"
"sort"
"strings"
"sync"
"syscall"
"time"
"highload/db/migrations"
"highload/internal/database/sqlc"
"highload/internal/model"
"github.com/google/uuid"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgxpool"
"github.com/jackc/pgx/v5/stdlib"
"github.com/pressly/goose/v3"
)
type recorder struct {
mu sync.Mutex
active bool
SQL string
args []any
}
func (r *recorder) TraceQueryStart(ctx context.Context, _ *pgx.Conn, data pgx.TraceQueryStartData) context.Context {
r.mu.Lock()
defer r.mu.Unlock()
if r.active {
r.SQL, r.args = data.SQL, append([]any(nil), data.Args...)
}
return ctx
}
func (*recorder) TraceQueryEnd(context.Context, *pgx.Conn, pgx.TraceQueryEndData) {}
type fixture struct {
admin *pgx.Conn
pool *pgxpool.Pool
schema string
owns bool
user string
chats []string
record recorder
researchBudget time.Duration
}
func (f *fixture) cleanup() error {
if f.pool != nil {
f.pool.Close()
}
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
var cleanupErr error
if f.admin != nil {
if f.owns {
_, cleanupErr = f.admin.Exec(ctx, "DROP SCHEMA "+pgx.Identifier{f.schema}.Sanitize()+" CASCADE")
}
_ = f.admin.Close(ctx)
}
return cleanupErr
}
func createFixture(ctx context.Context, url, schema string, rows int) (_ *fixture, err error) {
f := &fixture{schema: schema}
defer func() {
if err != nil {
f.cleanup()
}
}()
f.admin, err = pgx.Connect(ctx, url)
if err != nil {
return nil, err
}
_, err = f.admin.Exec(ctx, "CREATE SCHEMA "+pgx.Identifier{schema}.Sanitize())
if err != nil {
return nil, err
}
f.owns = true // Never drop a schema unless this invocation created it.
cfg, err := pgx.ParseConfig(url)
if err != nil {
return nil, err
}
cfg.RuntimeParams["search_path"] = schema
cfg.RuntimeParams["timezone"] = "UTC"
cfg.ConnectTimeout = 5 * time.Second
db := stdlib.OpenDB(*cfg)
provider, err := goose.NewProvider(goose.DialectPostgres, db, migrations.Files,
goose.WithVerbose(false), goose.WithDisableGlobalRegistry(true),
goose.WithSlog(slog.New(slog.NewJSONHandler(io.Discard, nil))))
if err == nil {
_, err = provider.Up(ctx)
}
db.Close()
if err != nil {
return nil, err
}
poolCfg, err := pgxpool.ParseConfig(url)
if err != nil {
return nil, err
}
poolCfg.MaxConns = 2
poolCfg.ConnConfig = cfg
poolCfg.ConnConfig.Tracer = &f.record
f.pool, err = pgxpool.NewWithConfig(ctx, poolCfg)
if err != nil {
return nil, err
}
f.user = uuid.NewString()
_, err = f.pool.Exec(ctx, "INSERT INTO users(id,username,display_name,password_hash) VALUES($1,$2,$3,$4)",
f.user, "sql_fixture", "Synthetic SQL fixture", "not-a-login-credential")
if err != nil {
return nil, err
}
for i := 0; i < 5; i++ {
id := uuid.NewString()
f.chats = append(f.chats, id)
if _, err = f.pool.Exec(ctx, "INSERT INTO chats(id,title) VALUES($1,$2)", id, fmt.Sprintf("SQL chat %d", i)); err != nil {
return nil, err
}
if _, err = f.pool.Exec(ctx, "INSERT INTO chat_members(chat_id,user_id) VALUES($1,$2)", id, f.user); err != nil {
return nil, err
}
}
_, err = f.pool.Exec(ctx, `INSERT INTO messages(chat_id,sender_id,text,created_at)
SELECT CASE WHEN n%100<70 THEN $1::uuid WHEN n%100<90 THEN $2::uuid
WHEN n%100<96 THEN $3::uuid WHEN n%100<99 THEN $4::uuid ELSE $5::uuid END,
$6::uuid, 'sql-fixture-'||n, timestamptz '2026-01-01 00:00:00+00'+n*interval '1 millisecond'
FROM generate_series(1,$7::integer) n`, f.chats[0], f.chats[1], f.chats[2], f.chats[3], f.chats[4], f.user, rows)
if err != nil {
return nil, err
}
_, err = f.pool.Exec(ctx, "ANALYZE messages")
if err != nil {
return nil, err
}
return f, nil
}
func (f *fixture) position(ctx context.Context) (*repository2.MessagePosition, error) {
var p repository2.MessagePosition
err := f.pool.QueryRow(ctx, `SELECT id::text,created_at FROM messages WHERE chat_id=$1
ORDER BY created_at,id OFFSET (SELECT count(*)*8/10 FROM messages WHERE chat_id=$1) LIMIT 1`, f.chats[0]).Scan(&p.ID, &p.CreatedAt)
// OFFSET is used only to select a fixture cursor, never in the measured query.
return &p, err
}
func (f *fixture) read(ctx context.Context, after *repository2.MessagePosition) (time.Duration, error) {
return f.readChat(ctx, f.chats[0], after)
}
func (f *fixture) readChat(ctx context.Context, chatID string, after *repository2.MessagePosition) (time.Duration, error) {
repo := postgres.NewMessageRepository(sqlc.New(f.pool))
f.record.mu.Lock()
f.record.active = true
f.record.mu.Unlock()
started := time.Now()
var items []*model.Message
var err error
if f.researchBudget > 0 {
items, err = f.researchRead(ctx, chatID, after)
} else {
items, err = repo.GetMessagesByChatID(ctx, chatID, 51, after)
}
elapsed := time.Since(started)
f.record.mu.Lock()
f.record.active = false
f.record.mu.Unlock()
if err == nil && len(items) != 51 {
err = fmt.Errorf("fixture history returned %d rows instead of 51", len(items))
}
for i := 1; err == nil && i < len(items); i++ {
if items[i].CreatedAt.Before(items[i-1].CreatedAt) {
err = errors.New("history out of order")
}
}
return elapsed, err
}
func maxVisited(node map[string]any) float64 {
number := func(key string) float64 { value, _ := node[key].(float64); return value }
visited := (number("Actual Rows") + number("Rows Removed by Filter") + number("Rows Removed by Index Recheck")) * number("Actual Loops")
children, _ := node["Plans"].([]any)
for _, child := range children {
if child, ok := child.(map[string]any); ok {
visited = max(visited, maxVisited(child))
}
}
return visited
}
func (f *fixture) plan(ctx context.Context, after *repository2.MessagePosition) (map[string]any, error) {
return f.planChat(ctx, f.chats[0], after)
}
func (f *fixture) planChat(ctx context.Context, chatID string, after *repository2.MessagePosition) (map[string]any, error) {
if _, err := f.readChat(ctx, chatID, after); err != nil {
return nil, err
}
f.record.mu.Lock()
query, args := f.record.SQL, append([]any(nil), f.record.args...)
f.record.mu.Unlock()
if !strings.Contains(query, "GetMessagesByChatID") {
return nil, errors.New("adapt query recorder to repository SQL")
}
var raw []byte
err := f.pool.QueryRow(ctx, "EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON) "+query, args...).Scan(&raw)
if err != nil {
return nil, err
}
var plan []map[string]any
if err = json.Unmarshal(raw, &plan); err != nil || len(plan) != 1 {
return nil, errors.New("invalid EXPLAIN output")
}
node, ok := plan[0]["Plan"].(map[string]any)
if !ok {
return nil, errors.New("missing EXPLAIN plan")
}
return map[string]any{"explain": plan[0], "max_visited_rows": maxVisited(node)}, nil
}
func latency(values []time.Duration) map[string]float64 {
sort.Slice(values, func(i, j int) bool { return values[i] < values[j] })
result := map[string]float64{}
for _, p := range []int{50, 95, 99} {
i := (len(values)*p+99)/100 - 1
result[fmt.Sprintf("p%d", p)] = float64(values[i].Microseconds()) / 1000
}
return result
}
func (f *fixture) compare(ctx context.Context, repetitions int) ([]map[string]any, error) {
position, err := f.position(ctx)
if err != nil {
return nil, err
}
// Only this fresh fixture schema is changed; application migrations stay intact.
if _, err = f.pool.Exec(ctx, "DROP INDEX idx_messages_chat_created_id"); err != nil {
return nil, err
}
var results []map[string]any
for _, columns := range []string{"chat_id", "chat_id,created_at,id"} {
if _, err = f.pool.Exec(ctx, "CREATE INDEX lab2_history_variant ON messages("+columns+")"); err != nil {
return nil, err
}
if _, err = f.pool.Exec(ctx, "ANALYZE messages"); err != nil {
return nil, err
}
for i := 0; i < 3; i++ {
if _, err = f.read(ctx, position); err != nil {
return nil, err
}
}
var reads, writes []time.Duration
for i := 0; i < repetitions; i++ {
duration, err := f.read(ctx, position)
if err != nil {
return nil, err
}
reads = append(reads, duration)
prefix := "write-fixture-" + uuid.NewString() + "-"
started := time.Now()
tx, err := f.pool.Begin(ctx)
if err != nil {
return nil, err
}
_, err = tx.Exec(ctx, `INSERT INTO messages(chat_id,sender_id,text)
SELECT $1::uuid,$2::uuid,$3::text||n FROM generate_series(1,200) n`, f.chats[0], f.user, prefix)
if err != nil {
_ = tx.Rollback(ctx)
return nil, err
}
if err = tx.Commit(ctx); err != nil {
return nil, err
}
writes = append(writes, time.Since(started))
if _, err = f.pool.Exec(ctx, "DELETE FROM messages WHERE text LIKE $1", prefix+"%"); err != nil {
return nil, err
}
}
first, err := f.plan(ctx, nil)
if err != nil {
return nil, err
}
deep, err := f.plan(ctx, position)
if err != nil {
return nil, err
}
var size int64
if err = f.pool.QueryRow(ctx, "SELECT pg_relation_size('lab2_history_variant')").Scan(&size); err != nil {
return nil, err
}
results = append(results, map[string]any{"columns": columns, "read_ms": latency(reads), "insert_200_with_commit_ms": latency(writes), "index_bytes": size, "first_page": first, "deep_page": deep})
if _, err = f.pool.Exec(ctx, "DROP INDEX lab2_history_variant"); err != nil {
return nil, err
}
}
return results, nil
}
func snapshot(ctx context.Context, url, schema string) (map[string]any, error) {
cfg, err := pgx.ParseConfig(url)
if err != nil {
return nil, err
}
cfg.RuntimeParams["timezone"] = "UTC"
cfg.RuntimeParams["datestyle"] = "ISO, YMD"
cfg.ConnectTimeout = 5 * time.Second
conn, err := pgx.ConnectConfig(ctx, cfg)
if err != nil {
return nil, err
}
defer conn.Close(context.Background())
tx, err := conn.BeginTx(ctx, pgx.TxOptions{IsoLevel: pgx.RepeatableRead, AccessMode: pgx.ReadOnly})
if err != nil {
return nil, err
}
defer tx.Rollback(context.Background())
identity := sha256.Sum256([]byte(fmt.Sprintf("%s:%d/%s", cfg.Host, cfg.Port, cfg.Database)))
tables := map[string]any{}
missing := 0
for _, table := range []struct{ name, order string }{{"users", "id"}, {"chats", "id"}, {"chat_members", "chat_id,user_id"}, {"messages", "id"}} {
qualified := pgx.Identifier{schema, table.name}.Sanitize()
var exists bool
if err = tx.QueryRow(ctx, "SELECT to_regclass($1) IS NOT NULL", qualified).Scan(&exists); err != nil {
return nil, err
}
if !exists {
missing++
continue
}
rows, err := tx.Query(ctx, "SELECT row_to_json(t)::text FROM "+qualified+" t ORDER BY "+table.order)
if err != nil {
return nil, err
}
digest := sha256.New()
count := 0
for rows.Next() {
var row string
if err = rows.Scan(&row); err != nil {
rows.Close()
return nil, err
}
_, _ = io.WriteString(digest, row+"\n")
count++
}
err = rows.Err()
rows.Close()
if err != nil {
return nil, err
}
tables[table.name] = map[string]any{"rows": count, "sha256": hex.EncodeToString(digest.Sum(nil))}
}
if missing != 0 && missing != 4 {
return nil, errors.New("snapshot schema contains only part of the application tables")
}
return map[string]any{"identity_sha256": hex.EncodeToString(identity[:]), "empty": missing == 4, "tables": tables}, nil
}
func execute(ctx context.Context, mode, url, schema string, rows, budget, repetitions int) (result map[string]any, err error) {
if mode == "snapshot" {
return snapshot(ctx, url, schema)
}
if rows < 10000 || ((mode == "compare" || mode == "seed") && rows < 100000) || repetitions < 2 || budget < 1 {
return nil, errors.New("invalid experiment limits")
}
if schema == "public" || mode != "seed" {
schema = "lab2_test_" + strings.ReplaceAll(uuid.NewString(), "-", "")
}
if !strings.HasPrefix(schema, "lab2_test_") {
return nil, errors.New("fixture schema must start with lab2_test_")
}
f, err := createFixture(ctx, url, schema, rows)
if err != nil {
return nil, err
}
defer func() {
if cleanupErr := f.cleanup(); err == nil {
err = cleanupErr
}
}()
result = map[string]any{"mode": mode, "rows": rows, "schema": schema, "chat_distribution_percent": []int{70, 20, 6, 3, 1}}
var actualRows int
var postgresVersion string
if err = f.pool.QueryRow(ctx, "SELECT count(*) FROM messages").Scan(&actualRows); err != nil || actualRows != rows {
return nil, errors.New("fixture row count differs from the requested dataset")
}
if err = f.pool.QueryRow(ctx, "SHOW server_version").Scan(&postgresVersion); err != nil {
return nil, err
}
distribution, queryErr := f.pool.Query(ctx, "SELECT count(*) FROM messages GROUP BY chat_id ORDER BY count(*) DESC")
if queryErr != nil {
return nil, queryErr
}
var chatCounts []int
for distribution.Next() {
var count int
if err = distribution.Scan(&count); err != nil {
distribution.Close()
return nil, err
}
chatCounts = append(chatCounts, count)
}
err = distribution.Err()
distribution.Close()
if err != nil || len(chatCounts) != 5 || chatCounts[0] <= chatCounts[4] {
return nil, errors.New("fixture does not demonstrate skewed chat sizes")
}
result["actual_rows"], result["chat_message_counts"] = actualRows, chatCounts
result["postgres_version"], result["go_version"] = postgresVersion, runtime.Version()
if mode == "seed" {
f.owns = false
result["kept_fixture"] = true
return result, nil
}
if mode == "compare" {
variants, err := f.compare(ctx, repetitions)
if err != nil {
return nil, err
}
result["variants"] = variants
result["limitations"] = "Warm single-host prototype; write cleanup can leave bloat. Repeat under fixed resources and review plans, cold/generic plans and loss boundaries."
return result, nil
}
if mode != "plan" {
return nil, errors.New("unknown mode")
}
position, err := f.position(ctx)
if err != nil {
return nil, err
}
first, err := f.plan(ctx, nil)
if err != nil {
return nil, err
}
deep, err := f.plan(ctx, position)
if err != nil {
return nil, err
}
if first["max_visited_rows"].(float64) > float64(budget) || deep["max_visited_rows"].(float64) > float64(budget) {
return nil, errors.New("history exceeds chosen scan budget")
}
result["first_page"], result["deep_page"], result["scan_budget_rows"] = first, deep, budget
return result, nil
}
func main() {
mode := flag.String("mode", "snapshot", "snapshot, plan, compare or seed")
schema := flag.String("schema", "public", "snapshot schema or fresh lab2_test_* seed schema")
rows := flag.Int("rows", 10000, "fixture message count")
budget := flag.Int("scan-budget", 1000, "engineering threshold, not a teacher-mandated constant")
repetitions := flag.Int("repetitions", 5, "measurement samples per variant")
series := flag.Int("series", 4, "independent AB/BA research rounds (even, >= 4)")
writeBatches := flag.Int("write-batches", 20, "committed batches of 200 rows per research trial")
timeout := flag.Duration("timeout", 3*time.Minute, "total SQL probe execution budget")
flag.Parse()
url := os.Getenv("LAB2_SQL_DATABASE_URL")
if url == "" {
fmt.Fprintln(os.Stderr, "set LAB2_SQL_DATABASE_URL for a dedicated test database")
os.Exit(2)
}
if *mode != "snapshot" && *mode != "plan" && *mode != "compare" && *mode != "seed" && *mode != "research" {
os.Exit(2)
}
signals, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
defer stop()
ctx, cancel := context.WithTimeout(signals, *timeout)
defer cancel()
var result map[string]any
var err error
if *mode == "research" {
result, err = research(ctx, url, *rows, *repetitions, *series, *writeBatches)
} else {
result, err = execute(ctx, *mode, url, *schema, *rows, *budget, *repetitions)
}
if err != nil {
fmt.Fprintln(os.Stderr, "SQL probe failed; private error details suppressed")
os.Exit(1)
}
if err = json.NewEncoder(os.Stdout).Encode(result); err != nil {
os.Exit(1)
}
}