344 lines
12 KiB
Go
344 lines
12 KiB
Go
// Command loadgen is the open-loop load generator of the experiments.
|
|
//
|
|
// loadgen bench — seeds data, warms up, then offers each rate of -rates
|
|
// for -step as a mix of history reads and message sends.
|
|
// loadgen timeline — one user sends messages at -rate, retrying uncertain
|
|
// sends with the same Idempotency-Key, then reads the
|
|
// history back: every confirmed message once.
|
|
package main
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"flag"
|
|
"fmt"
|
|
mrand "math/rand/v2"
|
|
"net/http"
|
|
"os"
|
|
"os/signal"
|
|
"path/filepath"
|
|
"strconv"
|
|
"strings"
|
|
"sync"
|
|
"syscall"
|
|
"time"
|
|
|
|
"github.com/google/uuid"
|
|
)
|
|
|
|
type config struct {
|
|
Mode string `json:"mode"`
|
|
BaseURL string `json:"base_url"`
|
|
Users int `json:"users"`
|
|
Chats int `json:"chats"`
|
|
SeedMessages int `json:"seed_messages_per_chat"`
|
|
ReadRatio float64 `json:"read_ratio"`
|
|
Rates []float64 `json:"rates"`
|
|
Step string `json:"step"`
|
|
Warmup string `json:"warmup"`
|
|
Rate float64 `json:"rate"`
|
|
Duration string `json:"duration"`
|
|
Timeout string `json:"client_timeout"`
|
|
Seed uint64 `json:"seed"`
|
|
Scrape bool `json:"scrape"`
|
|
StopOnFail bool `json:"stop_on_fail"`
|
|
StatsURL string `json:"stats_url"`
|
|
APIHost string `json:"api_host"`
|
|
DebugPort string `json:"debug_port"`
|
|
Out string `json:"-"`
|
|
|
|
step, warmup, duration, timeout time.Duration
|
|
}
|
|
|
|
func main() {
|
|
if len(os.Args) < 2 || (os.Args[1] != "bench" && os.Args[1] != "timeline") {
|
|
fmt.Fprintln(os.Stderr, "usage: loadgen bench|timeline [flags]; loadgen bench -h for the flags")
|
|
os.Exit(2)
|
|
}
|
|
cfg, err := parse(os.Args[1], os.Args[2:])
|
|
if err != nil {
|
|
fmt.Fprintln(os.Stderr, "loadgen:", err)
|
|
os.Exit(2)
|
|
}
|
|
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
|
|
defer stop()
|
|
if cfg.Mode == "bench" {
|
|
err = runBench(ctx, cfg)
|
|
} else {
|
|
err = runTimeline(ctx, cfg)
|
|
}
|
|
if err != nil {
|
|
fmt.Fprintln(os.Stderr, "loadgen:", err)
|
|
os.Exit(1)
|
|
}
|
|
}
|
|
|
|
func parse(mode string, args []string) (config, error) {
|
|
cfg := config{Mode: mode}
|
|
fs := flag.NewFlagSet(mode, flag.ContinueOnError)
|
|
rates := fs.String("rates", "200,400", "bench: offered requests per second of each step")
|
|
fs.StringVar(&cfg.BaseURL, "base-url", "http://lb:8080", "entry point of the API")
|
|
fs.IntVar(&cfg.Users, "users", 200, "bench: synthetic users")
|
|
fs.IntVar(&cfg.Chats, "chats", 100, "bench: chats of 2-5 random members")
|
|
fs.IntVar(&cfg.SeedMessages, "seed-messages", 50, "bench: messages written into each chat before measuring")
|
|
fs.Float64Var(&cfg.ReadRatio, "read-ratio", 0.8, "bench: share of history reads; the rest are message sends")
|
|
fs.StringVar(&cfg.Step, "step", "30s", "bench: duration of each step")
|
|
fs.StringVar(&cfg.Warmup, "warmup", "30s", "bench: unmeasured warm-up at the first rate")
|
|
fs.Float64Var(&cfg.Rate, "rate", 50, "timeline: messages per second")
|
|
fs.StringVar(&cfg.Duration, "duration", "60s", "timeline: duration")
|
|
fs.StringVar(&cfg.Timeout, "timeout", "2s", "how long the generator waits for a response (class client_timeout)")
|
|
fs.Uint64Var(&cfg.Seed, "seed", 1, "random seed of the request mix")
|
|
fs.BoolVar(&cfg.StopOnFail, "stop-on-fail", false, "bench: end the ladder after the first step over 1 % errors or 500 ms p99")
|
|
fs.BoolVar(&cfg.Scrape, "scrape", false, "sample HAProxy statistics and replica expvar every second")
|
|
fs.StringVar(&cfg.StatsURL, "stats-url", "http://lb:8404/;csv", "HAProxy statistics CSV")
|
|
fs.StringVar(&cfg.APIHost, "api-host", "api", "DNS name of the API replicas")
|
|
fs.StringVar(&cfg.DebugPort, "debug-port", "9090", "internal expvar port of the replicas (APP_DEBUG_ADDR)")
|
|
fs.StringVar(&cfg.Out, "out", "/out", "directory for the results")
|
|
if err := fs.Parse(args); err != nil {
|
|
return cfg, err
|
|
}
|
|
for _, part := range strings.Split(*rates, ",") {
|
|
r, err := strconv.ParseFloat(strings.TrimSpace(part), 64)
|
|
if err != nil || r <= 0 {
|
|
return cfg, fmt.Errorf("rates: %q is not a positive number", part)
|
|
}
|
|
cfg.Rates = append(cfg.Rates, r)
|
|
}
|
|
for _, d := range []struct {
|
|
name string
|
|
raw string
|
|
dst *time.Duration
|
|
}{{"step", cfg.Step, &cfg.step}, {"warmup", cfg.Warmup, &cfg.warmup}, {"duration", cfg.Duration, &cfg.duration}, {"timeout", cfg.Timeout, &cfg.timeout}} {
|
|
v, err := time.ParseDuration(d.raw)
|
|
if err != nil || v < 0 {
|
|
return cfg, fmt.Errorf("%s: %q is not a duration", d.name, d.raw)
|
|
}
|
|
*d.dst = v
|
|
}
|
|
if cfg.ReadRatio < 0 || cfg.ReadRatio > 1 || cfg.Rate <= 0 || cfg.step <= 0 || cfg.timeout <= 0 {
|
|
return cfg, fmt.Errorf("read-ratio must be within 0..1; rate, step and timeout positive")
|
|
}
|
|
return cfg, os.MkdirAll(cfg.Out, 0o755)
|
|
}
|
|
|
|
func writeJSON(path string, v any) error {
|
|
raw, err := json.MarshalIndent(v, "", " ")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return os.WriteFile(path, append(raw, '\n'), 0o644)
|
|
}
|
|
|
|
// startScrape returns the function that stops the sampling and waits.
|
|
func startScrape(ctx context.Context, cfg config, start time.Time) func() {
|
|
if !cfg.Scrape {
|
|
return func() {}
|
|
}
|
|
ctx, cancel := context.WithCancel(ctx)
|
|
s := scraper{statsURL: cfg.StatsURL, apiHost: cfg.APIHost, debugPort: cfg.DebugPort, http: &http.Client{}}
|
|
var wg sync.WaitGroup
|
|
wg.Go(func() {
|
|
if err := s.run(ctx, start, filepath.Join(cfg.Out, "metrics.jsonl")); err != nil {
|
|
fmt.Fprintln(os.Stderr, "loadgen: scrape:", err)
|
|
}
|
|
})
|
|
return func() { cancel(); wg.Wait() }
|
|
}
|
|
|
|
type benchReport struct {
|
|
StartedAt time.Time `json:"started_at"`
|
|
Config config `json:"config"`
|
|
Users int `json:"users"`
|
|
Chats int `json:"chats"`
|
|
Seeded int `json:"seeded_messages"`
|
|
Steps []stepSummary `json:"steps"`
|
|
WritesOK int `json:"messages_written_during_steps"`
|
|
}
|
|
|
|
func runBench(ctx context.Context, cfg config) error {
|
|
c := newClient(cfg.BaseURL, cfg.timeout)
|
|
rng := mrand.New(mrand.NewPCG(cfg.Seed, 0x6c6f6164))
|
|
ds, err := seed(ctx, newClient(cfg.BaseURL, seedTimeout), rng, cfg.Users, cfg.Chats, cfg.SeedMessages)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
report := benchReport{StartedAt: time.Now().UTC(), Config: cfg, Users: len(ds.actors), Chats: len(ds.rooms), Seeded: ds.messages}
|
|
start := time.Now()
|
|
stopScrape := startScrape(ctx, cfg, start)
|
|
defer stopScrape()
|
|
|
|
// next runs on the scheduling goroutine, which owns the random source.
|
|
counter := 0
|
|
next := func(rec *recorder) func(time.Time) func() {
|
|
return func(planned time.Time) func() {
|
|
r := ds.rooms[rng.IntN(len(ds.rooms))]
|
|
a := ds.actors[r.members[rng.IntN(len(r.members))]]
|
|
write := rng.Float64() >= cfg.ReadRatio
|
|
counter++
|
|
text := fmt.Sprintf("load %d", counter)
|
|
return func() {
|
|
sent := time.Now()
|
|
var res result
|
|
if write {
|
|
res = c.send(ctx, r.id, a.token, uuid.NewString(), text)
|
|
} else {
|
|
res = c.firstPage(ctx, r.id, a.token)
|
|
}
|
|
rec.add(sample{planned: planned, lag: sent.Sub(planned), latency: res.latency, class: classify(res), instance: res.instance, write: write})
|
|
}
|
|
}
|
|
}
|
|
if cfg.warmup > 0 {
|
|
openLoop(ctx, cfg.Rates[0], cfg.warmup, next(&recorder{}))
|
|
}
|
|
all := &recorder{}
|
|
for _, rate := range cfg.Rates {
|
|
if ctx.Err() != nil {
|
|
break
|
|
}
|
|
stepStart := time.Now()
|
|
rec := &recorder{}
|
|
openLoop(ctx, rate, cfg.step, next(rec))
|
|
s := summarize(rate, cfg.step, rec.all())
|
|
s.StartedAt = stepStart.Sub(start).Seconds()
|
|
report.Steps = append(report.Steps, s)
|
|
report.WritesOK += s.Writes
|
|
for _, x := range rec.all() {
|
|
all.add(x)
|
|
}
|
|
fmt.Printf("step %6.0f rps: ok %7.1f rps, errors %5.2f%%, p99 %7.1f ms, lag p99 %5.1f ms, valid %v, passed %v\n",
|
|
rate, s.OKRPS, 100*s.ErrorShare, s.P99Ms, s.LagP99Ms, s.Valid, s.Passed)
|
|
if cfg.StopOnFail && s.overloaded() {
|
|
break
|
|
}
|
|
}
|
|
stopScrape()
|
|
if err := os.WriteFile(filepath.Join(cfg.Out, "seconds.csv"), []byte(secondsCSV(start, all.all())), 0o644); err != nil {
|
|
return err
|
|
}
|
|
return writeJSON(filepath.Join(cfg.Out, "steps.json"), report)
|
|
}
|
|
|
|
type timelineReport struct {
|
|
StartedAt time.Time `json:"started_at"`
|
|
Config config `json:"config"`
|
|
Jobs int `json:"messages_attempted"`
|
|
Attempts int `json:"requests"`
|
|
Confirmed int `json:"confirmed"`
|
|
RetriedJobs int `json:"messages_retried"`
|
|
Unresolved int `json:"unresolved"`
|
|
Rejected int `json:"rejected_4xx"`
|
|
Classes map[string]int `json:"classes"`
|
|
Instances map[string]int `json:"instances"`
|
|
History int `json:"history_messages"`
|
|
Duplicates int `json:"duplicates"`
|
|
Missing int `json:"missing_confirmed"`
|
|
Passed bool `json:"passed"`
|
|
}
|
|
|
|
// retryDelay spaces the attempts of one message whose outcome was uncertain.
|
|
const retryDelay = 200 * time.Millisecond
|
|
|
|
// retryGrace bounds the retries of an uncertain message after the schedule ends.
|
|
const retryGrace = 10 * time.Second
|
|
|
|
func runTimeline(ctx context.Context, cfg config) error {
|
|
c := newClient(cfg.BaseURL, cfg.timeout)
|
|
rng := mrand.New(mrand.NewPCG(cfg.Seed, 0x74696d65))
|
|
ds, err := seed(ctx, newClient(cfg.BaseURL, seedTimeout), rng, 2, 1, 0)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
r := ds.rooms[0]
|
|
sender := ds.actors[r.members[0]]
|
|
report := timelineReport{StartedAt: time.Now().UTC(), Config: cfg, Classes: map[string]int{}, Instances: map[string]int{}}
|
|
runID := randomHex(3)
|
|
start := time.Now()
|
|
stopScrape := startScrape(ctx, cfg, start)
|
|
defer stopScrape()
|
|
deadline := start.Add(cfg.duration + retryGrace)
|
|
|
|
var mu sync.Mutex
|
|
confirmed := map[string]string{} // text -> message id
|
|
rec := &recorder{}
|
|
jobs := 0
|
|
openLoop(ctx, cfg.Rate, cfg.duration, func(planned time.Time) func() {
|
|
jobs++
|
|
key, text := uuid.NewString(), fmt.Sprintf("timeline %s %d", runID, jobs)
|
|
return func() {
|
|
for attempt := 0; ; attempt++ {
|
|
sent := time.Now()
|
|
res := c.send(ctx, r.id, sender.token, key, text)
|
|
cls := classify(res)
|
|
rec.add(sample{planned: planned, lag: sent.Sub(planned), latency: res.latency, class: cls, instance: res.instance, write: true})
|
|
mu.Lock()
|
|
switch {
|
|
case cls == "ok":
|
|
var m message
|
|
if json.Unmarshal(res.body, &m) == nil {
|
|
confirmed[text] = m.ID
|
|
}
|
|
case res.err == nil && res.status >= 400 && res.status < 500:
|
|
report.Rejected++
|
|
case time.Now().After(deadline) || ctx.Err() != nil:
|
|
report.Unresolved++
|
|
default:
|
|
if attempt == 0 {
|
|
report.RetriedJobs++
|
|
}
|
|
mu.Unlock()
|
|
time.Sleep(retryDelay)
|
|
planned = time.Now()
|
|
continue
|
|
}
|
|
mu.Unlock()
|
|
return
|
|
}
|
|
}
|
|
})
|
|
stopScrape()
|
|
samples := rec.all()
|
|
report.Jobs, report.Attempts, report.Confirmed = jobs, len(samples), len(confirmed)
|
|
for _, x := range samples {
|
|
report.Classes[x.class]++
|
|
if x.instance != "" {
|
|
report.Instances[x.instance]++
|
|
}
|
|
}
|
|
|
|
history, err := c.history(ctx, r.id, sender.token)
|
|
if err != nil {
|
|
return fmt.Errorf("read history back: %w", err)
|
|
}
|
|
report.History = len(history)
|
|
seen := map[string]int{}
|
|
byID := map[string]string{}
|
|
for _, m := range history {
|
|
seen[m.Text]++
|
|
byID[m.ID] = m.Text
|
|
}
|
|
for _, n := range seen {
|
|
if n > 1 {
|
|
report.Duplicates += n - 1
|
|
}
|
|
}
|
|
for text, id := range confirmed {
|
|
if byID[id] != text {
|
|
report.Missing++
|
|
}
|
|
}
|
|
report.Passed = report.Duplicates == 0 && report.Missing == 0
|
|
if err := os.WriteFile(filepath.Join(cfg.Out, "seconds.csv"), []byte(secondsCSV(start, samples)), 0o644); err != nil {
|
|
return err
|
|
}
|
|
if err := writeJSON(filepath.Join(cfg.Out, "timeline.json"), report); err != nil {
|
|
return err
|
|
}
|
|
fmt.Printf("timeline: %d messages, %d confirmed, %d retried, %d unresolved, %d duplicates, %d missing\n",
|
|
report.Jobs, report.Confirmed, report.RetriedJobs, report.Unresolved, report.Duplicates, report.Missing)
|
|
if !report.Passed {
|
|
return fmt.Errorf("history check failed: %d duplicates, %d confirmed messages missing", report.Duplicates, report.Missing)
|
|
}
|
|
return nil
|
|
}
|