Files
mtusi2026lab3/student-tests/lab2/sqlprobe/research.go
T

256 lines
9.8 KiB
Go

package main
import (
"context"
"crypto/sha256"
"encoding/hex"
"errors"
"fmt"
"os"
"strings"
"time"
"github.com/google/uuid"
"github.com/jackc/pgx/v5/pgconn"
"github.com/jackc/pgx/v5/pgtype"
"highload/internal/database/sqlc"
"highload/internal/model"
repository2 "highload/internal/repository"
)
// Research reuses the actual repository SQL. No application index is changed:
// the fixture owns a fresh schema and removes it when the experiment finishes.
func research(ctx context.Context, url string, rows, repetitions, series, batches int) (result map[string]any, err error) {
stage := "validate"
defer func() {
if err != nil {
fmt.Fprintln(os.Stderr, "research: failed at "+stage)
var pgErr *pgconn.PgError
if errors.As(err, &pgErr) {
fmt.Fprintln(os.Stderr, "sqlstate: "+pgErr.Code)
}
if errors.Is(err, context.DeadlineExceeded) {
fmt.Fprintln(os.Stderr, "research: deadline exceeded")
}
if errors.Is(err, repository2.ErrServiceUnavailable) {
fmt.Fprintln(os.Stderr, "research: repository unavailable")
}
for _, text := range []string{"timeout", "context deadline exceeded", "invalid input", "unexpected page", "history out of order"} {
if strings.Contains(err.Error(), text) {
fmt.Fprintln(os.Stderr, "research: "+text)
}
}
if strings.HasPrefix(err.Error(), "fixture history returned ") {
fmt.Fprintln(os.Stderr, "research: "+err.Error())
}
}
}()
if rows < 100000 || rows%100 != 0 || repetitions < 30 || series < 4 || series%2 != 0 || batches < 10 {
return nil, errors.New("research needs >=100k rows, >=30 reads, even >=4 series, >=10 write batches")
}
schema := "lab2_test_" + uuid.NewString()[:8] + "_research"
stage = "create-fixture"
f, err := createFixture(ctx, url, schema, rows)
if err != nil {
return nil, err
}
// Preserve the API's one-second policy. Only the offline SQL experiment uses
// a larger, identical budget for both indexes, so slow baselines are measurable.
f.researchBudget = 10 * time.Second
defer func() {
if cleanupErr := f.cleanup(); err == nil {
err = cleanupErr
}
}()
stage = "drop-original-index"
if _, err = f.pool.Exec(ctx, "DROP INDEX idx_messages_chat_created_id"); err != nil {
return nil, err
}
// Disable autovacuum only for the disposable table to control trial cleanup.
if _, err = f.pool.Exec(ctx, "ALTER TABLE messages SET (autovacuum_enabled=false)"); err != nil {
return nil, err
}
// Fixed message body size; index keys and logical rows remain identical in A/B.
stage = "payload"
if _, err = f.pool.Exec(ctx, "UPDATE messages SET text=rpad(text,256,'x')"); err != nil {
return nil, err
}
var version string
if err = f.pool.QueryRow(ctx, "SHOW server_version").Scan(&version); err != nil {
return nil, err
}
type readCase struct {
name, chat string
after *repository2.MessagePosition
}
var cases []readCase
for _, item := range []struct {
name string
index int
}{{"large", 0}, {"medium", 2}, {"small", 4}} {
stage = "cursor-" + item.name
chatID := f.chats[item.index]
position := &repository2.MessagePosition{}
if err = f.pool.QueryRow(ctx, `SELECT id::text,created_at FROM messages WHERE chat_id=$1
ORDER BY created_at,id OFFSET (SELECT count(*)*8/10 FROM messages WHERE chat_id=$1) LIMIT 1`, chatID).Scan(&position.ID, &position.CreatedAt); err != nil {
return nil, err
}
cases = append(cases, readCase{item.name + "_first", chatID, nil}, readCase{item.name + "_deep", chatID, position})
}
stage = "initial-digest"
before, err := f.messageDigest(ctx)
if err != nil {
return nil, err
}
var trials []map[string]any
for round := 0; round < series; round++ {
order := []string{"A", "B"}
if round%2 == 1 {
order = []string{"B", "A"}
}
for ordinal, variant := range order {
stage = "vacuum-" + variant
// Remove dead tuples and reset physical bloat between trials, outside timing.
if _, err = f.pool.Exec(ctx, "VACUUM (FULL, ANALYZE) messages"); err != nil {
return nil, err
}
columns := "chat_id"
if variant == "B" {
columns = "chat_id,created_at,id"
}
buildStart := time.Now()
stage = "build-index-" + variant
if _, err = f.pool.Exec(ctx, "CREATE INDEX lab2_history_variant ON messages("+columns+")"); err != nil {
return nil, err
}
buildMS := float64(time.Since(buildStart).Microseconds()) / 1000
if _, err = f.pool.Exec(ctx, "ANALYZE messages"); err != nil {
return nil, err
}
var indexBytes, tableBytes int64
if err = f.pool.QueryRow(ctx, "SELECT pg_relation_size('lab2_history_variant'),pg_relation_size('messages')").Scan(&indexBytes, &tableBytes); err != nil {
return nil, err
}
readResults := map[string]any{}
for _, c := range cases {
stage = "warmup-" + variant + "-" + c.name
for warmup := 0; warmup < 5; warmup++ {
if _, err = f.readChat(ctx, c.chat, c.after); err != nil {
return nil, err
}
}
var timings []time.Duration
var raw []float64
started := time.Now()
stage = "read-" + variant + "-" + c.name
for n := 0; n < repetitions; n++ {
d, readErr := f.readChat(ctx, c.chat, c.after)
if readErr != nil {
return nil, readErr
}
timings = append(timings, d)
raw = append(raw, float64(d.Microseconds())/1000)
}
elapsed := time.Since(started).Seconds()
stage = "plan-" + variant + "-" + c.name
plan, planErr := f.planChat(ctx, c.chat, c.after)
if planErr != nil {
return nil, planErr
}
readResults[c.name] = map[string]any{"samples_ms": raw, "latency_ms": latency(timings), "successful_rps": float64(repetitions) / elapsed, "elapsed_seconds": elapsed, "plan": plan}
}
var writes []time.Duration
var writeMS []float64
var walBytes []int64
for n := 0; n < batches; n++ {
stage = "write-" + variant
var lsn string
if err = f.pool.QueryRow(ctx, "SELECT pg_current_wal_insert_lsn()::text").Scan(&lsn); err != nil {
return nil, err
}
started := time.Now()
tx, txErr := f.pool.Begin(ctx)
if txErr != nil {
return nil, txErr
}
_, txErr = tx.Exec(ctx, `INSERT INTO messages(chat_id,sender_id,text)
SELECT $1::uuid,$2::uuid,rpad('research-write-'||n,256,'x') FROM generate_series(1,200) n`, f.chats[0], f.user)
if txErr == nil {
txErr = tx.Commit(ctx)
} else {
_ = tx.Rollback(context.Background())
}
if txErr != nil {
return nil, txErr
}
d := time.Since(started)
writes = append(writes, d)
writeMS = append(writeMS, float64(d.Microseconds())/1000)
var delta int64
if err = f.pool.QueryRow(ctx, "SELECT pg_wal_lsn_diff(pg_current_wal_insert_lsn(),$1::pg_lsn)::bigint", lsn).Scan(&delta); err != nil {
return nil, err
}
walBytes = append(walBytes, delta)
}
stage = "delete-fixture-writes-" + variant
if _, err = f.pool.Exec(ctx, "DELETE FROM messages WHERE text LIKE 'research-write-%'"); err != nil {
return nil, err
}
stage = "verify-dataset-" + variant
after, digestErr := f.messageDigest(ctx)
if digestErr != nil || after != before {
return nil, errors.New("logical dataset changed between trials")
}
trials = append(trials, map[string]any{"series": round + 1, "order": ordinal + 1, "variant": variant, "columns": columns, "read_cases": readResults, "index_bytes": indexBytes, "table_bytes": tableBytes, "build_ms": buildMS, "insert_200_with_commit_ms": latency(writes), "write_samples_ms": writeMS, "write_wal_bytes": walBytes, "dataset_sha256": before})
fmt.Fprintf(os.Stderr, "series %d variant %s complete\n", round+1, variant)
stage = "drop-index-" + variant
if _, err = f.pool.Exec(ctx, "DROP INDEX lab2_history_variant"); err != nil {
return nil, err
}
}
}
return map[string]any{"mode": "research", "rows": rows, "chat_counts": []int{rows * 70 / 100, rows * 20 / 100, rows * 6 / 100, rows * 3 / 100, rows / 100}, "payload_bytes": 256, "postgres_version": version, "series": series, "reads_per_case": repetitions, "write_batches_per_trial": batches, "rows_per_write_batch": 200, "warmup_per_case": 5, "concurrent_clients": 1, "sql_experiment_timeout_seconds": 10, "application_repository_timeout_seconds": 1, "dataset_sha256": before, "trials": trials, "limitations": "Warm sequential closed-loop SQL through the same generated sqlc query; no HTTP/KDF. Offline query budget 10s, application repository unchanged at 1s. Same logical rows, VACUUM FULL and rebuild before trials. pgx prepared-plan behavior is retained. WAL is cluster-wide on a dedicated quiet instance; sampled CPU includes administration. No cold-cache or sustained-capacity claim."}, nil
}
func (f *fixture) researchRead(ctx context.Context, chatID string, after *repository2.MessagePosition) ([]*model.Message, error) {
queryCtx, cancel := context.WithTimeout(ctx, f.researchBudget)
defer cancel()
var id pgtype.UUID
if err := id.Scan(chatID); err != nil {
return nil, err
}
params := sqlc.GetMessagesByChatIDParams{ChatID: id, LimitCount: 51}
if after != nil {
params.AfterCreatedAt = pgtype.Timestamptz{Time: after.CreatedAt, Valid: true}
if err := params.AfterID.Scan(after.ID); err != nil {
return nil, err
}
}
rows, err := sqlc.New(f.pool).GetMessagesByChatID(queryCtx, params)
if err != nil {
return nil, err
}
items := make([]*model.Message, len(rows))
for i, m := range rows {
items[i] = &model.Message{ID: m.ID.String(), ChatID: m.ChatID.String(), SenderID: m.SenderID.String(), Text: m.Text, CreatedAt: m.CreatedAt.Time}
}
return items, nil
}
func (f *fixture) messageDigest(ctx context.Context) (string, error) {
rows, err := f.pool.Query(ctx, "SELECT row_to_json(m)::text FROM messages m ORDER BY id")
if err != nil {
return "", err
}
defer rows.Close()
h := sha256.New()
for rows.Next() {
var line string
if err = rows.Scan(&line); err != nil {
return "", err
}
_, _ = fmt.Fprintln(h, line)
}
return hex.EncodeToString(h.Sum(nil)), rows.Err()
}