package main import ( "context" "errors" "fmt" "math" "net" "slices" "sort" "strings" "time" ) // maxValidLag: if requests left later than this at p99, the generator, not // the system, limited the step. const maxValidLag = 10 * time.Millisecond // The capacity criterion of the study: a step passes when it is valid, has // fewer than 1 % errors and its successful requests have a p99 within 500 ms. const ( maxErrorShare = 0.01 sloP99 = 500 * time.Millisecond ) // sample is one request as the generator saw it. type sample struct { planned time.Time lag time.Duration // actual send time minus planned send time latency time.Duration class string instance string write bool } // classify names the outcome of a request. Errors of the balancer itself carry // no X-Instance-Id and get the suffix _lb; client_timeout is the generator // giving up, not a timeout of the system. func classify(r result) string { switch { case r.err != nil && isTimeout(r.err): return "client_timeout" case r.err != nil: return "conn_error" case r.status >= 200 && r.status < 300: return "ok" case r.instance == "": return fmt.Sprintf("http_%d_lb", r.status) default: return fmt.Sprintf("http_%d", r.status) } } func isTimeout(err error) bool { if errors.Is(err, context.DeadlineExceeded) { return true } var netErr net.Error return errors.As(err, &netErr) && netErr.Timeout() } // percentile is the nearest-rank percentile of an ascending slice. func percentile(sorted []time.Duration, p float64) time.Duration { if len(sorted) == 0 { return 0 } rank := int(math.Ceil(p / 100 * float64(len(sorted)))) return sorted[min(max(rank, 1), len(sorted))-1] } type stepSummary struct { OfferedRPS float64 `json:"offered_rps"` Seconds float64 `json:"seconds"` StartedAt float64 `json:"started_at_s"` Sent int `json:"sent"` OK int `json:"ok"` OKRPS float64 `json:"ok_rps"` ErrorShare float64 `json:"error_share"` Classes map[string]int `json:"classes"` P50Ms float64 `json:"ok_p50_ms"` P95Ms float64 `json:"ok_p95_ms"` P99Ms float64 `json:"ok_p99_ms"` LagP99Ms float64 `json:"lag_p99_ms"` Valid bool `json:"valid"` Passed bool `json:"passed"` Instances map[string]int `json:"instances"` Writes int `json:"writes_ok"` } // summarize reduces one step; latency percentiles cover successful requests. func summarize(rate float64, d time.Duration, samples []sample) stepSummary { s := stepSummary{ OfferedRPS: rate, Seconds: d.Seconds(), Sent: len(samples), Classes: map[string]int{}, Instances: map[string]int{}, } var ok, lags []time.Duration for _, x := range samples { s.Classes[x.class]++ lags = append(lags, x.lag) if x.instance != "" { s.Instances[x.instance]++ } if x.class == "ok" { ok = append(ok, x.latency) if x.write { s.Writes++ } } } slices.Sort(ok) slices.Sort(lags) s.OK = len(ok) s.OKRPS = float64(s.OK) / d.Seconds() if s.Sent > 0 { s.ErrorShare = float64(s.Sent-s.OK) / float64(s.Sent) } s.P50Ms, s.P95Ms, s.P99Ms = ms(percentile(ok, 50)), ms(percentile(ok, 95)), ms(percentile(ok, 99)) lagP99 := percentile(lags, 99) s.LagP99Ms = ms(lagP99) s.Valid = lagP99 <= maxValidLag s.Passed = s.Valid && !s.overloaded() return s } func (s stepSummary) overloaded() bool { return s.ErrorShare >= maxErrorShare || s.P99Ms > ms(sloP99) } func ms(d time.Duration) float64 { return float64(d.Microseconds()) / 1000 } // errorColumns are the fixed error columns of the per-second CSV; any other // class lands in err_other. var errorColumns = []string{"client_timeout", "conn_error"} // secondsCSV renders the per-second timeline, attributed by planned send // time so it lines up with the moment a fault was injected. func secondsCSV(start time.Time, samples []sample) string { type bucket struct { samples []sample } buckets := map[int]*bucket{} last := 0 for _, x := range samples { sec := int(x.planned.Sub(start) / time.Second) if buckets[sec] == nil { buckets[sec] = &bucket{} } buckets[sec].samples = append(buckets[sec].samples, x) last = max(last, sec) } var b strings.Builder b.WriteString("second,sent,ok,client_timeout,conn_error,http_5xx_api,http_5xx_lb,http_4xx,err_other,ok_p50_ms,ok_p95_ms,ok_p99_ms,lag_p99_ms,instances\n") for sec := 0; sec <= last; sec++ { bk := buckets[sec] if bk == nil { fmt.Fprintf(&b, "%d,0,0,0,0,0,0,0,0,0,0,0,0,\n", sec) continue } var ok, lags []time.Duration counts := map[string]int{} instances := map[string]int{} for _, x := range bk.samples { lags = append(lags, x.lag) if x.instance != "" { instances[x.instance]++ } switch { case x.class == "ok": ok = append(ok, x.latency) case slices.Contains(errorColumns, x.class): counts[x.class]++ case strings.HasPrefix(x.class, "http_5") && strings.HasSuffix(x.class, "_lb"): counts["http_5xx_lb"]++ case strings.HasPrefix(x.class, "http_5"): counts["http_5xx_api"]++ case strings.HasPrefix(x.class, "http_4"): counts["http_4xx"]++ default: counts["err_other"]++ } } slices.Sort(ok) slices.Sort(lags) names := make([]string, 0, len(instances)) for name := range instances { names = append(names, name) } sort.Strings(names) parts := make([]string, 0, len(names)) for _, name := range names { parts = append(parts, fmt.Sprintf("%s=%d", name, instances[name])) } fmt.Fprintf(&b, "%d,%d,%d,%d,%d,%d,%d,%d,%d,%.1f,%.1f,%.1f,%.1f,%s\n", sec, len(bk.samples), len(ok), counts["client_timeout"], counts["conn_error"], counts["http_5xx_api"], counts["http_5xx_lb"], counts["http_4xx"], counts["err_other"], ms(percentile(ok, 50)), ms(percentile(ok, 95)), ms(percentile(ok, 99)), ms(percentile(lags, 99)), strings.Join(parts, "|")) } return b.String() }