// Command loadgen is the open-loop load generator of the experiments. // // loadgen bench — seeds data, warms up, then offers each rate of -rates // for -step as a mix of history reads and message sends. // loadgen timeline — one user sends messages at -rate, retrying uncertain // sends with the same Idempotency-Key, then reads the // history back: every confirmed message once. package main import ( "context" "encoding/json" "flag" "fmt" mrand "math/rand/v2" "net/http" "os" "os/signal" "path/filepath" "strconv" "strings" "sync" "syscall" "time" "github.com/google/uuid" ) type config struct { Mode string `json:"mode"` BaseURL string `json:"base_url"` Users int `json:"users"` Chats int `json:"chats"` SeedMessages int `json:"seed_messages_per_chat"` ReadRatio float64 `json:"read_ratio"` Rates []float64 `json:"rates"` Step string `json:"step"` Warmup string `json:"warmup"` Rate float64 `json:"rate"` Duration string `json:"duration"` Timeout string `json:"client_timeout"` Seed uint64 `json:"seed"` Scrape bool `json:"scrape"` StopOnFail bool `json:"stop_on_fail"` StatsURL string `json:"stats_url"` APIHost string `json:"api_host"` DebugPort string `json:"debug_port"` Out string `json:"-"` step, warmup, duration, timeout time.Duration } func main() { if len(os.Args) < 2 || (os.Args[1] != "bench" && os.Args[1] != "timeline") { fmt.Fprintln(os.Stderr, "usage: loadgen bench|timeline [flags]; loadgen bench -h for the flags") os.Exit(2) } cfg, err := parse(os.Args[1], os.Args[2:]) if err != nil { fmt.Fprintln(os.Stderr, "loadgen:", err) os.Exit(2) } ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) defer stop() if cfg.Mode == "bench" { err = runBench(ctx, cfg) } else { err = runTimeline(ctx, cfg) } if err != nil { fmt.Fprintln(os.Stderr, "loadgen:", err) os.Exit(1) } } func parse(mode string, args []string) (config, error) { cfg := config{Mode: mode} fs := flag.NewFlagSet(mode, flag.ContinueOnError) rates := fs.String("rates", "200,400", "bench: offered requests per second of each step") fs.StringVar(&cfg.BaseURL, "base-url", "http://lb:8080", "entry point of the API") fs.IntVar(&cfg.Users, "users", 200, "bench: synthetic users") fs.IntVar(&cfg.Chats, "chats", 100, "bench: chats of 2-5 random members") fs.IntVar(&cfg.SeedMessages, "seed-messages", 50, "bench: messages written into each chat before measuring") fs.Float64Var(&cfg.ReadRatio, "read-ratio", 0.8, "bench: share of history reads; the rest are message sends") fs.StringVar(&cfg.Step, "step", "30s", "bench: duration of each step") fs.StringVar(&cfg.Warmup, "warmup", "30s", "bench: unmeasured warm-up at the first rate") fs.Float64Var(&cfg.Rate, "rate", 50, "timeline: messages per second") fs.StringVar(&cfg.Duration, "duration", "60s", "timeline: duration") fs.StringVar(&cfg.Timeout, "timeout", "2s", "how long the generator waits for a response (class client_timeout)") fs.Uint64Var(&cfg.Seed, "seed", 1, "random seed of the request mix") fs.BoolVar(&cfg.StopOnFail, "stop-on-fail", false, "bench: end the ladder after the first step over 1 % errors or 500 ms p99") fs.BoolVar(&cfg.Scrape, "scrape", false, "sample HAProxy statistics and replica expvar every second") fs.StringVar(&cfg.StatsURL, "stats-url", "http://lb:8404/;csv", "HAProxy statistics CSV") fs.StringVar(&cfg.APIHost, "api-host", "api", "DNS name of the API replicas") fs.StringVar(&cfg.DebugPort, "debug-port", "9090", "internal expvar port of the replicas (APP_DEBUG_ADDR)") fs.StringVar(&cfg.Out, "out", "/out", "directory for the results") if err := fs.Parse(args); err != nil { return cfg, err } for _, part := range strings.Split(*rates, ",") { r, err := strconv.ParseFloat(strings.TrimSpace(part), 64) if err != nil || r <= 0 { return cfg, fmt.Errorf("rates: %q is not a positive number", part) } cfg.Rates = append(cfg.Rates, r) } for _, d := range []struct { name string raw string dst *time.Duration }{{"step", cfg.Step, &cfg.step}, {"warmup", cfg.Warmup, &cfg.warmup}, {"duration", cfg.Duration, &cfg.duration}, {"timeout", cfg.Timeout, &cfg.timeout}} { v, err := time.ParseDuration(d.raw) if err != nil || v < 0 { return cfg, fmt.Errorf("%s: %q is not a duration", d.name, d.raw) } *d.dst = v } if cfg.ReadRatio < 0 || cfg.ReadRatio > 1 || cfg.Rate <= 0 || cfg.step <= 0 || cfg.timeout <= 0 { return cfg, fmt.Errorf("read-ratio must be within 0..1; rate, step and timeout positive") } return cfg, os.MkdirAll(cfg.Out, 0o755) } func writeJSON(path string, v any) error { raw, err := json.MarshalIndent(v, "", " ") if err != nil { return err } return os.WriteFile(path, append(raw, '\n'), 0o644) } // startScrape returns the function that stops the sampling and waits. func startScrape(ctx context.Context, cfg config, start time.Time) func() { if !cfg.Scrape { return func() {} } ctx, cancel := context.WithCancel(ctx) s := scraper{statsURL: cfg.StatsURL, apiHost: cfg.APIHost, debugPort: cfg.DebugPort, http: &http.Client{}} var wg sync.WaitGroup wg.Go(func() { if err := s.run(ctx, start, filepath.Join(cfg.Out, "metrics.jsonl")); err != nil { fmt.Fprintln(os.Stderr, "loadgen: scrape:", err) } }) return func() { cancel(); wg.Wait() } } type benchReport struct { StartedAt time.Time `json:"started_at"` Config config `json:"config"` Users int `json:"users"` Chats int `json:"chats"` Seeded int `json:"seeded_messages"` Steps []stepSummary `json:"steps"` WritesOK int `json:"messages_written_during_steps"` } func runBench(ctx context.Context, cfg config) error { c := newClient(cfg.BaseURL, cfg.timeout) rng := mrand.New(mrand.NewPCG(cfg.Seed, 0x6c6f6164)) ds, err := seed(ctx, newClient(cfg.BaseURL, seedTimeout), rng, cfg.Users, cfg.Chats, cfg.SeedMessages) if err != nil { return err } report := benchReport{StartedAt: time.Now().UTC(), Config: cfg, Users: len(ds.actors), Chats: len(ds.rooms), Seeded: ds.messages} start := time.Now() stopScrape := startScrape(ctx, cfg, start) defer stopScrape() // next runs on the scheduling goroutine, which owns the random source. counter := 0 next := func(rec *recorder) func(time.Time) func() { return func(planned time.Time) func() { r := ds.rooms[rng.IntN(len(ds.rooms))] a := ds.actors[r.members[rng.IntN(len(r.members))]] write := rng.Float64() >= cfg.ReadRatio counter++ text := fmt.Sprintf("load %d", counter) return func() { sent := time.Now() var res result if write { res = c.send(ctx, r.id, a.token, uuid.NewString(), text) } else { res = c.firstPage(ctx, r.id, a.token) } rec.add(sample{planned: planned, lag: sent.Sub(planned), latency: res.latency, class: classify(res), instance: res.instance, write: write}) } } } if cfg.warmup > 0 { openLoop(ctx, cfg.Rates[0], cfg.warmup, next(&recorder{})) } all := &recorder{} for _, rate := range cfg.Rates { if ctx.Err() != nil { break } stepStart := time.Now() rec := &recorder{} openLoop(ctx, rate, cfg.step, next(rec)) s := summarize(rate, cfg.step, rec.all()) s.StartedAt = stepStart.Sub(start).Seconds() report.Steps = append(report.Steps, s) report.WritesOK += s.Writes for _, x := range rec.all() { all.add(x) } fmt.Printf("step %6.0f rps: ok %7.1f rps, errors %5.2f%%, p99 %7.1f ms, lag p99 %5.1f ms, valid %v, passed %v\n", rate, s.OKRPS, 100*s.ErrorShare, s.P99Ms, s.LagP99Ms, s.Valid, s.Passed) if cfg.StopOnFail && s.overloaded() { break } } stopScrape() if err := os.WriteFile(filepath.Join(cfg.Out, "seconds.csv"), []byte(secondsCSV(start, all.all())), 0o644); err != nil { return err } return writeJSON(filepath.Join(cfg.Out, "steps.json"), report) } type timelineReport struct { StartedAt time.Time `json:"started_at"` Config config `json:"config"` Jobs int `json:"messages_attempted"` Attempts int `json:"requests"` Confirmed int `json:"confirmed"` RetriedJobs int `json:"messages_retried"` Unresolved int `json:"unresolved"` Rejected int `json:"rejected_4xx"` Classes map[string]int `json:"classes"` Instances map[string]int `json:"instances"` History int `json:"history_messages"` Duplicates int `json:"duplicates"` Missing int `json:"missing_confirmed"` Passed bool `json:"passed"` } // retryDelay spaces the attempts of one message whose outcome was uncertain. const retryDelay = 200 * time.Millisecond // retryGrace bounds the retries of an uncertain message after the schedule ends. const retryGrace = 10 * time.Second func runTimeline(ctx context.Context, cfg config) error { c := newClient(cfg.BaseURL, cfg.timeout) rng := mrand.New(mrand.NewPCG(cfg.Seed, 0x74696d65)) ds, err := seed(ctx, newClient(cfg.BaseURL, seedTimeout), rng, 2, 1, 0) if err != nil { return err } r := ds.rooms[0] sender := ds.actors[r.members[0]] report := timelineReport{StartedAt: time.Now().UTC(), Config: cfg, Classes: map[string]int{}, Instances: map[string]int{}} runID := randomHex(3) start := time.Now() stopScrape := startScrape(ctx, cfg, start) defer stopScrape() deadline := start.Add(cfg.duration + retryGrace) var mu sync.Mutex confirmed := map[string]string{} // text -> message id rec := &recorder{} jobs := 0 openLoop(ctx, cfg.Rate, cfg.duration, func(planned time.Time) func() { jobs++ key, text := uuid.NewString(), fmt.Sprintf("timeline %s %d", runID, jobs) return func() { for attempt := 0; ; attempt++ { sent := time.Now() res := c.send(ctx, r.id, sender.token, key, text) cls := classify(res) rec.add(sample{planned: planned, lag: sent.Sub(planned), latency: res.latency, class: cls, instance: res.instance, write: true}) mu.Lock() switch { case cls == "ok": var m message if json.Unmarshal(res.body, &m) == nil { confirmed[text] = m.ID } case res.err == nil && res.status >= 400 && res.status < 500: report.Rejected++ case time.Now().After(deadline) || ctx.Err() != nil: report.Unresolved++ default: if attempt == 0 { report.RetriedJobs++ } mu.Unlock() time.Sleep(retryDelay) planned = time.Now() continue } mu.Unlock() return } } }) stopScrape() samples := rec.all() report.Jobs, report.Attempts, report.Confirmed = jobs, len(samples), len(confirmed) for _, x := range samples { report.Classes[x.class]++ if x.instance != "" { report.Instances[x.instance]++ } } history, err := c.history(ctx, r.id, sender.token) if err != nil { return fmt.Errorf("read history back: %w", err) } report.History = len(history) seen := map[string]int{} byID := map[string]string{} for _, m := range history { seen[m.Text]++ byID[m.ID] = m.Text } for _, n := range seen { if n > 1 { report.Duplicates += n - 1 } } for text, id := range confirmed { if byID[id] != text { report.Missing++ } } report.Passed = report.Duplicates == 0 && report.Missing == 0 if err := os.WriteFile(filepath.Join(cfg.Out, "seconds.csv"), []byte(secondsCSV(start, samples)), 0o644); err != nil { return err } if err := writeJSON(filepath.Join(cfg.Out, "timeline.json"), report); err != nil { return err } fmt.Printf("timeline: %d messages, %d confirmed, %d retried, %d unresolved, %d duplicates, %d missing\n", report.Jobs, report.Confirmed, report.RetriedJobs, report.Unresolved, report.Duplicates, report.Missing) if !report.Passed { return fmt.Errorf("history check failed: %d duplicates, %d confirmed messages missing", report.Duplicates, report.Missing) } return nil }