206 lines
5.9 KiB
Go
206 lines
5.9 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"math"
|
|
"net"
|
|
"slices"
|
|
"sort"
|
|
"strings"
|
|
"time"
|
|
)
|
|
|
|
// maxValidLag: if requests left later than this at p99, the generator, not
|
|
// the system, limited the step.
|
|
const maxValidLag = 10 * time.Millisecond
|
|
|
|
// The capacity criterion of the study: a step passes when it is valid, has
|
|
// fewer than 1 % errors and its successful requests have a p99 within 500 ms.
|
|
const (
|
|
maxErrorShare = 0.01
|
|
sloP99 = 500 * time.Millisecond
|
|
)
|
|
|
|
// sample is one request as the generator saw it.
|
|
type sample struct {
|
|
planned time.Time
|
|
lag time.Duration // actual send time minus planned send time
|
|
latency time.Duration
|
|
class string
|
|
instance string
|
|
write bool
|
|
}
|
|
|
|
// classify names the outcome of a request. Errors of the balancer itself carry
|
|
// no X-Instance-Id and get the suffix _lb; client_timeout is the generator
|
|
// giving up, not a timeout of the system.
|
|
func classify(r result) string {
|
|
switch {
|
|
case r.err != nil && isTimeout(r.err):
|
|
return "client_timeout"
|
|
case r.err != nil:
|
|
return "conn_error"
|
|
case r.status >= 200 && r.status < 300:
|
|
return "ok"
|
|
case r.instance == "":
|
|
return fmt.Sprintf("http_%d_lb", r.status)
|
|
default:
|
|
return fmt.Sprintf("http_%d", r.status)
|
|
}
|
|
}
|
|
|
|
func isTimeout(err error) bool {
|
|
if errors.Is(err, context.DeadlineExceeded) {
|
|
return true
|
|
}
|
|
var netErr net.Error
|
|
return errors.As(err, &netErr) && netErr.Timeout()
|
|
}
|
|
|
|
// percentile is the nearest-rank percentile of an ascending slice.
|
|
func percentile(sorted []time.Duration, p float64) time.Duration {
|
|
if len(sorted) == 0 {
|
|
return 0
|
|
}
|
|
rank := int(math.Ceil(p / 100 * float64(len(sorted))))
|
|
return sorted[min(max(rank, 1), len(sorted))-1]
|
|
}
|
|
|
|
type stepSummary struct {
|
|
OfferedRPS float64 `json:"offered_rps"`
|
|
Seconds float64 `json:"seconds"`
|
|
StartedAt float64 `json:"started_at_s"`
|
|
Sent int `json:"sent"`
|
|
OK int `json:"ok"`
|
|
OKRPS float64 `json:"ok_rps"`
|
|
ErrorShare float64 `json:"error_share"`
|
|
Classes map[string]int `json:"classes"`
|
|
P50Ms float64 `json:"ok_p50_ms"`
|
|
P95Ms float64 `json:"ok_p95_ms"`
|
|
P99Ms float64 `json:"ok_p99_ms"`
|
|
LagP99Ms float64 `json:"lag_p99_ms"`
|
|
Valid bool `json:"valid"`
|
|
Passed bool `json:"passed"`
|
|
Instances map[string]int `json:"instances"`
|
|
Writes int `json:"writes_ok"`
|
|
}
|
|
|
|
// summarize reduces one step; latency percentiles cover successful requests.
|
|
func summarize(rate float64, d time.Duration, samples []sample) stepSummary {
|
|
s := stepSummary{
|
|
OfferedRPS: rate,
|
|
Seconds: d.Seconds(),
|
|
Sent: len(samples),
|
|
Classes: map[string]int{},
|
|
Instances: map[string]int{},
|
|
}
|
|
var ok, lags []time.Duration
|
|
for _, x := range samples {
|
|
s.Classes[x.class]++
|
|
lags = append(lags, x.lag)
|
|
if x.instance != "" {
|
|
s.Instances[x.instance]++
|
|
}
|
|
if x.class == "ok" {
|
|
ok = append(ok, x.latency)
|
|
if x.write {
|
|
s.Writes++
|
|
}
|
|
}
|
|
}
|
|
slices.Sort(ok)
|
|
slices.Sort(lags)
|
|
s.OK = len(ok)
|
|
s.OKRPS = float64(s.OK) / d.Seconds()
|
|
if s.Sent > 0 {
|
|
s.ErrorShare = float64(s.Sent-s.OK) / float64(s.Sent)
|
|
}
|
|
s.P50Ms, s.P95Ms, s.P99Ms = ms(percentile(ok, 50)), ms(percentile(ok, 95)), ms(percentile(ok, 99))
|
|
lagP99 := percentile(lags, 99)
|
|
s.LagP99Ms = ms(lagP99)
|
|
s.Valid = lagP99 <= maxValidLag
|
|
s.Passed = s.Valid && !s.overloaded()
|
|
return s
|
|
}
|
|
|
|
func (s stepSummary) overloaded() bool {
|
|
return s.ErrorShare >= maxErrorShare || s.P99Ms > ms(sloP99)
|
|
}
|
|
|
|
func ms(d time.Duration) float64 {
|
|
return float64(d.Microseconds()) / 1000
|
|
}
|
|
|
|
// errorColumns are the fixed error columns of the per-second CSV; any other
|
|
// class lands in err_other.
|
|
var errorColumns = []string{"client_timeout", "conn_error"}
|
|
|
|
// secondsCSV renders the per-second timeline, attributed by planned send
|
|
// time so it lines up with the moment a fault was injected.
|
|
func secondsCSV(start time.Time, samples []sample) string {
|
|
type bucket struct {
|
|
samples []sample
|
|
}
|
|
buckets := map[int]*bucket{}
|
|
last := 0
|
|
for _, x := range samples {
|
|
sec := int(x.planned.Sub(start) / time.Second)
|
|
if buckets[sec] == nil {
|
|
buckets[sec] = &bucket{}
|
|
}
|
|
buckets[sec].samples = append(buckets[sec].samples, x)
|
|
last = max(last, sec)
|
|
}
|
|
var b strings.Builder
|
|
b.WriteString("second,sent,ok,client_timeout,conn_error,http_5xx_api,http_5xx_lb,http_4xx,err_other,ok_p50_ms,ok_p95_ms,ok_p99_ms,lag_p99_ms,instances\n")
|
|
for sec := 0; sec <= last; sec++ {
|
|
bk := buckets[sec]
|
|
if bk == nil {
|
|
fmt.Fprintf(&b, "%d,0,0,0,0,0,0,0,0,0,0,0,0,\n", sec)
|
|
continue
|
|
}
|
|
var ok, lags []time.Duration
|
|
counts := map[string]int{}
|
|
instances := map[string]int{}
|
|
for _, x := range bk.samples {
|
|
lags = append(lags, x.lag)
|
|
if x.instance != "" {
|
|
instances[x.instance]++
|
|
}
|
|
switch {
|
|
case x.class == "ok":
|
|
ok = append(ok, x.latency)
|
|
case slices.Contains(errorColumns, x.class):
|
|
counts[x.class]++
|
|
case strings.HasPrefix(x.class, "http_5") && strings.HasSuffix(x.class, "_lb"):
|
|
counts["http_5xx_lb"]++
|
|
case strings.HasPrefix(x.class, "http_5"):
|
|
counts["http_5xx_api"]++
|
|
case strings.HasPrefix(x.class, "http_4"):
|
|
counts["http_4xx"]++
|
|
default:
|
|
counts["err_other"]++
|
|
}
|
|
}
|
|
slices.Sort(ok)
|
|
slices.Sort(lags)
|
|
names := make([]string, 0, len(instances))
|
|
for name := range instances {
|
|
names = append(names, name)
|
|
}
|
|
sort.Strings(names)
|
|
parts := make([]string, 0, len(names))
|
|
for _, name := range names {
|
|
parts = append(parts, fmt.Sprintf("%s=%d", name, instances[name]))
|
|
}
|
|
fmt.Fprintf(&b, "%d,%d,%d,%d,%d,%d,%d,%d,%d,%.1f,%.1f,%.1f,%.1f,%s\n",
|
|
sec, len(bk.samples), len(ok),
|
|
counts["client_timeout"], counts["conn_error"], counts["http_5xx_api"], counts["http_5xx_lb"], counts["http_4xx"], counts["err_other"],
|
|
ms(percentile(ok, 50)), ms(percentile(ok, 95)), ms(percentile(ok, 99)), ms(percentile(lags, 99)),
|
|
strings.Join(parts, "|"))
|
|
}
|
|
return b.String()
|
|
}
|