Files
mtusi2026lab5/tools/loadgen/main.go
T

344 lines
12 KiB
Go

// Command loadgen is the open-loop load generator of the experiments.
//
// loadgen bench — seeds data, warms up, then offers each rate of -rates
// for -step as a mix of history reads and message sends.
// loadgen timeline — one user sends messages at -rate, retrying uncertain
// sends with the same Idempotency-Key, then reads the
// history back: every confirmed message once.
package main
import (
"context"
"encoding/json"
"flag"
"fmt"
mrand "math/rand/v2"
"net/http"
"os"
"os/signal"
"path/filepath"
"strconv"
"strings"
"sync"
"syscall"
"time"
"github.com/google/uuid"
)
type config struct {
Mode string `json:"mode"`
BaseURL string `json:"base_url"`
Users int `json:"users"`
Chats int `json:"chats"`
SeedMessages int `json:"seed_messages_per_chat"`
ReadRatio float64 `json:"read_ratio"`
Rates []float64 `json:"rates"`
Step string `json:"step"`
Warmup string `json:"warmup"`
Rate float64 `json:"rate"`
Duration string `json:"duration"`
Timeout string `json:"client_timeout"`
Seed uint64 `json:"seed"`
Scrape bool `json:"scrape"`
StopOnFail bool `json:"stop_on_fail"`
StatsURL string `json:"stats_url"`
APIHost string `json:"api_host"`
DebugPort string `json:"debug_port"`
Out string `json:"-"`
step, warmup, duration, timeout time.Duration
}
func main() {
if len(os.Args) < 2 || (os.Args[1] != "bench" && os.Args[1] != "timeline") {
fmt.Fprintln(os.Stderr, "usage: loadgen bench|timeline [flags]; loadgen bench -h for the flags")
os.Exit(2)
}
cfg, err := parse(os.Args[1], os.Args[2:])
if err != nil {
fmt.Fprintln(os.Stderr, "loadgen:", err)
os.Exit(2)
}
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
defer stop()
if cfg.Mode == "bench" {
err = runBench(ctx, cfg)
} else {
err = runTimeline(ctx, cfg)
}
if err != nil {
fmt.Fprintln(os.Stderr, "loadgen:", err)
os.Exit(1)
}
}
func parse(mode string, args []string) (config, error) {
cfg := config{Mode: mode}
fs := flag.NewFlagSet(mode, flag.ContinueOnError)
rates := fs.String("rates", "200,400", "bench: offered requests per second of each step")
fs.StringVar(&cfg.BaseURL, "base-url", "http://lb:8080", "entry point of the API")
fs.IntVar(&cfg.Users, "users", 200, "bench: synthetic users")
fs.IntVar(&cfg.Chats, "chats", 100, "bench: chats of 2-5 random members")
fs.IntVar(&cfg.SeedMessages, "seed-messages", 50, "bench: messages written into each chat before measuring")
fs.Float64Var(&cfg.ReadRatio, "read-ratio", 0.8, "bench: share of history reads; the rest are message sends")
fs.StringVar(&cfg.Step, "step", "30s", "bench: duration of each step")
fs.StringVar(&cfg.Warmup, "warmup", "30s", "bench: unmeasured warm-up at the first rate")
fs.Float64Var(&cfg.Rate, "rate", 50, "timeline: messages per second")
fs.StringVar(&cfg.Duration, "duration", "60s", "timeline: duration")
fs.StringVar(&cfg.Timeout, "timeout", "2s", "how long the generator waits for a response (class client_timeout)")
fs.Uint64Var(&cfg.Seed, "seed", 1, "random seed of the request mix")
fs.BoolVar(&cfg.StopOnFail, "stop-on-fail", false, "bench: end the ladder after the first step over 1 % errors or 500 ms p99")
fs.BoolVar(&cfg.Scrape, "scrape", false, "sample HAProxy statistics and replica expvar every second")
fs.StringVar(&cfg.StatsURL, "stats-url", "http://lb:8404/;csv", "HAProxy statistics CSV")
fs.StringVar(&cfg.APIHost, "api-host", "api", "DNS name of the API replicas")
fs.StringVar(&cfg.DebugPort, "debug-port", "9090", "internal expvar port of the replicas (APP_DEBUG_ADDR)")
fs.StringVar(&cfg.Out, "out", "/out", "directory for the results")
if err := fs.Parse(args); err != nil {
return cfg, err
}
for _, part := range strings.Split(*rates, ",") {
r, err := strconv.ParseFloat(strings.TrimSpace(part), 64)
if err != nil || r <= 0 {
return cfg, fmt.Errorf("rates: %q is not a positive number", part)
}
cfg.Rates = append(cfg.Rates, r)
}
for _, d := range []struct {
name string
raw string
dst *time.Duration
}{{"step", cfg.Step, &cfg.step}, {"warmup", cfg.Warmup, &cfg.warmup}, {"duration", cfg.Duration, &cfg.duration}, {"timeout", cfg.Timeout, &cfg.timeout}} {
v, err := time.ParseDuration(d.raw)
if err != nil || v < 0 {
return cfg, fmt.Errorf("%s: %q is not a duration", d.name, d.raw)
}
*d.dst = v
}
if cfg.ReadRatio < 0 || cfg.ReadRatio > 1 || cfg.Rate <= 0 || cfg.step <= 0 || cfg.timeout <= 0 {
return cfg, fmt.Errorf("read-ratio must be within 0..1; rate, step and timeout positive")
}
return cfg, os.MkdirAll(cfg.Out, 0o755)
}
func writeJSON(path string, v any) error {
raw, err := json.MarshalIndent(v, "", " ")
if err != nil {
return err
}
return os.WriteFile(path, append(raw, '\n'), 0o644)
}
// startScrape returns the function that stops the sampling and waits.
func startScrape(ctx context.Context, cfg config, start time.Time) func() {
if !cfg.Scrape {
return func() {}
}
ctx, cancel := context.WithCancel(ctx)
s := scraper{statsURL: cfg.StatsURL, apiHost: cfg.APIHost, debugPort: cfg.DebugPort, http: &http.Client{}}
var wg sync.WaitGroup
wg.Go(func() {
if err := s.run(ctx, start, filepath.Join(cfg.Out, "metrics.jsonl")); err != nil {
fmt.Fprintln(os.Stderr, "loadgen: scrape:", err)
}
})
return func() { cancel(); wg.Wait() }
}
type benchReport struct {
StartedAt time.Time `json:"started_at"`
Config config `json:"config"`
Users int `json:"users"`
Chats int `json:"chats"`
Seeded int `json:"seeded_messages"`
Steps []stepSummary `json:"steps"`
WritesOK int `json:"messages_written_during_steps"`
}
func runBench(ctx context.Context, cfg config) error {
c := newClient(cfg.BaseURL, cfg.timeout)
rng := mrand.New(mrand.NewPCG(cfg.Seed, 0x6c6f6164))
ds, err := seed(ctx, newClient(cfg.BaseURL, seedTimeout), rng, cfg.Users, cfg.Chats, cfg.SeedMessages)
if err != nil {
return err
}
report := benchReport{StartedAt: time.Now().UTC(), Config: cfg, Users: len(ds.actors), Chats: len(ds.rooms), Seeded: ds.messages}
start := time.Now()
stopScrape := startScrape(ctx, cfg, start)
defer stopScrape()
// next runs on the scheduling goroutine, which owns the random source.
counter := 0
next := func(rec *recorder) func(time.Time) func() {
return func(planned time.Time) func() {
r := ds.rooms[rng.IntN(len(ds.rooms))]
a := ds.actors[r.members[rng.IntN(len(r.members))]]
write := rng.Float64() >= cfg.ReadRatio
counter++
text := fmt.Sprintf("load %d", counter)
return func() {
sent := time.Now()
var res result
if write {
res = c.send(ctx, r.id, a.token, uuid.NewString(), text)
} else {
res = c.firstPage(ctx, r.id, a.token)
}
rec.add(sample{planned: planned, lag: sent.Sub(planned), latency: res.latency, class: classify(res), instance: res.instance, write: write})
}
}
}
if cfg.warmup > 0 {
openLoop(ctx, cfg.Rates[0], cfg.warmup, next(&recorder{}))
}
all := &recorder{}
for _, rate := range cfg.Rates {
if ctx.Err() != nil {
break
}
stepStart := time.Now()
rec := &recorder{}
openLoop(ctx, rate, cfg.step, next(rec))
s := summarize(rate, cfg.step, rec.all())
s.StartedAt = stepStart.Sub(start).Seconds()
report.Steps = append(report.Steps, s)
report.WritesOK += s.Writes
for _, x := range rec.all() {
all.add(x)
}
fmt.Printf("step %6.0f rps: ok %7.1f rps, errors %5.2f%%, p99 %7.1f ms, lag p99 %5.1f ms, valid %v, passed %v\n",
rate, s.OKRPS, 100*s.ErrorShare, s.P99Ms, s.LagP99Ms, s.Valid, s.Passed)
if cfg.StopOnFail && s.overloaded() {
break
}
}
stopScrape()
if err := os.WriteFile(filepath.Join(cfg.Out, "seconds.csv"), []byte(secondsCSV(start, all.all())), 0o644); err != nil {
return err
}
return writeJSON(filepath.Join(cfg.Out, "steps.json"), report)
}
type timelineReport struct {
StartedAt time.Time `json:"started_at"`
Config config `json:"config"`
Jobs int `json:"messages_attempted"`
Attempts int `json:"requests"`
Confirmed int `json:"confirmed"`
RetriedJobs int `json:"messages_retried"`
Unresolved int `json:"unresolved"`
Rejected int `json:"rejected_4xx"`
Classes map[string]int `json:"classes"`
Instances map[string]int `json:"instances"`
History int `json:"history_messages"`
Duplicates int `json:"duplicates"`
Missing int `json:"missing_confirmed"`
Passed bool `json:"passed"`
}
// retryDelay spaces the attempts of one message whose outcome was uncertain.
const retryDelay = 200 * time.Millisecond
// retryGrace bounds the retries of an uncertain message after the schedule ends.
const retryGrace = 10 * time.Second
func runTimeline(ctx context.Context, cfg config) error {
c := newClient(cfg.BaseURL, cfg.timeout)
rng := mrand.New(mrand.NewPCG(cfg.Seed, 0x74696d65))
ds, err := seed(ctx, newClient(cfg.BaseURL, seedTimeout), rng, 2, 1, 0)
if err != nil {
return err
}
r := ds.rooms[0]
sender := ds.actors[r.members[0]]
report := timelineReport{StartedAt: time.Now().UTC(), Config: cfg, Classes: map[string]int{}, Instances: map[string]int{}}
runID := randomHex(3)
start := time.Now()
stopScrape := startScrape(ctx, cfg, start)
defer stopScrape()
deadline := start.Add(cfg.duration + retryGrace)
var mu sync.Mutex
confirmed := map[string]string{} // text -> message id
rec := &recorder{}
jobs := 0
openLoop(ctx, cfg.Rate, cfg.duration, func(planned time.Time) func() {
jobs++
key, text := uuid.NewString(), fmt.Sprintf("timeline %s %d", runID, jobs)
return func() {
for attempt := 0; ; attempt++ {
sent := time.Now()
res := c.send(ctx, r.id, sender.token, key, text)
cls := classify(res)
rec.add(sample{planned: planned, lag: sent.Sub(planned), latency: res.latency, class: cls, instance: res.instance, write: true})
mu.Lock()
switch {
case cls == "ok":
var m message
if json.Unmarshal(res.body, &m) == nil {
confirmed[text] = m.ID
}
case res.err == nil && res.status >= 400 && res.status < 500:
report.Rejected++
case time.Now().After(deadline) || ctx.Err() != nil:
report.Unresolved++
default:
if attempt == 0 {
report.RetriedJobs++
}
mu.Unlock()
time.Sleep(retryDelay)
planned = time.Now()
continue
}
mu.Unlock()
return
}
}
})
stopScrape()
samples := rec.all()
report.Jobs, report.Attempts, report.Confirmed = jobs, len(samples), len(confirmed)
for _, x := range samples {
report.Classes[x.class]++
if x.instance != "" {
report.Instances[x.instance]++
}
}
history, err := c.history(ctx, r.id, sender.token)
if err != nil {
return fmt.Errorf("read history back: %w", err)
}
report.History = len(history)
seen := map[string]int{}
byID := map[string]string{}
for _, m := range history {
seen[m.Text]++
byID[m.ID] = m.Text
}
for _, n := range seen {
if n > 1 {
report.Duplicates += n - 1
}
}
for text, id := range confirmed {
if byID[id] != text {
report.Missing++
}
}
report.Passed = report.Duplicates == 0 && report.Missing == 0
if err := os.WriteFile(filepath.Join(cfg.Out, "seconds.csv"), []byte(secondsCSV(start, samples)), 0o644); err != nil {
return err
}
if err := writeJSON(filepath.Join(cfg.Out, "timeline.json"), report); err != nil {
return err
}
fmt.Printf("timeline: %d messages, %d confirmed, %d retried, %d unresolved, %d duplicates, %d missing\n",
report.Jobs, report.Confirmed, report.RetriedJobs, report.Unresolved, report.Duplicates, report.Missing)
if !report.Passed {
return fmt.Errorf("history check failed: %d duplicates, %d confirmed messages missing", report.Duplicates, report.Missing)
}
return nil
}