From f5f8040783c5c2f30b6e430b09fa05a81c5e655e Mon Sep 17 00:00:00 2001 From: kudesn1k Date: Wed, 4 Nov 2026 12:58:00 +0300 Subject: [PATCH] =?UTF-8?q?=D0=98=D1=81=D0=BF=D1=80=D0=B0=D0=B2=D0=B8?= =?UTF-8?q?=D1=82=D1=8C=20=D0=B7=D0=B0=D0=BC=D0=B5=D1=87=D0=B0=D0=BD=D0=B8?= =?UTF-8?q?=D1=8F=20=D1=80=D0=B5=D0=B2=D1=8C=D1=8E:=20=D0=BE=D1=81=D1=82?= =?UTF-8?q?=D0=B0=D0=BD=D0=B0=D0=B2=D0=BB=D0=B8=D0=B2=D0=B0=D1=82=D1=8C=20?= =?UTF-8?q?=D0=BB=D0=B5=D1=81=D1=82=D0=BD=D0=B8=D1=86=D1=83=20=D1=82=D0=BE?= =?UTF-8?q?=D0=BB=D1=8C=D0=BA=D0=BE=20=D0=BF=D1=80=D0=B8=20=D0=BF=D0=B5?= =?UTF-8?q?=D1=80=D0=B5=D0=B3=D1=80=D1=83=D0=B7=D0=BA=D0=B5=20=D1=81=D0=B8?= =?UTF-8?q?=D1=81=D1=82=D0=B5=D0=BC=D1=8B,=20=D0=BF=D1=80=D0=BE=D0=B2?= =?UTF-8?q?=D0=B5=D1=80=D1=8F=D1=82=D1=8C=20=D1=84=D0=B0=D0=B9=D0=BB=D1=8B?= =?UTF-8?q?=20=D1=81=D0=B5=D1=80=D0=B8=D0=B9=20=D0=B4=D0=BE=20=D0=B7=D0=B0?= =?UTF-8?q?=D0=BF=D1=83=D1=81=D0=BA=D0=B0,=20=D0=B8=D1=81=D0=BF=D1=80?= =?UTF-8?q?=D0=B0=D0=B2=D0=B8=D1=82=D1=8C=20=D0=B5=D0=B4=D0=B8=D0=BD=D0=B8?= =?UTF-8?q?=D1=86=D1=8B=20=D0=BA=D0=BE=D0=BD=D1=82=D1=80=D0=BE=D0=BB=D1=8F?= =?UTF-8?q?=20=D1=88=D1=83=D0=BC=D0=B0,=20=D1=83=D0=B1=D1=80=D0=B0=D1=82?= =?UTF-8?q?=D1=8C=20=D0=BB=D0=B8=D1=88=D0=BD=D0=B8=D0=B5=20=D0=BA=D0=BE?= =?UTF-8?q?=D0=BC=D0=BC=D0=B5=D0=BD=D1=82=D0=B0=D1=80=D0=B8=D0=B8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- infra/haproxy/haproxy.cfg | 2 +- internal/platform/config/config.go | 5 +---- scripts/bench/lib.sh | 3 ++- scripts/bench/plot.py | 35 +++++++++++++++++++----------- scripts/bench/run-series.sh | 5 +++++ tools/loadgen/main.go | 4 ++-- tools/loadgen/stats.go | 6 ++++- tools/loadgen/stats_test.go | 20 +++++++++++++++++ 8 files changed, 58 insertions(+), 22 deletions(-) diff --git a/infra/haproxy/haproxy.cfg b/infra/haproxy/haproxy.cfg index 7cb20e25..129badc3 100644 --- a/infra/haproxy/haproxy.cfg +++ b/infra/haproxy/haproxy.cfg @@ -61,7 +61,7 @@ backend replicas http-check expect status 200 # Out within 1.5 s (inter + fastinter), less than the API's 2 s drain delay. # maxconn 20: the replica's 10 pooled database connections plus requests - # that need no database; the rest waits in the queue. 0 means no limit. + # that need no database; the rest waits in the queue. default-server maxconn "${API_MAXCONN-20}" inter 1s fastinter 500ms downinter 1s fall 2 rise 2 check resolvers docker init-addr none resolve-prefer ipv4 # Up to five replicas, doubled while docker rollout replaces them. server-template api 10 api:8080 diff --git a/internal/platform/config/config.go b/internal/platform/config/config.go index 49e191f3..936b8ddc 100644 --- a/internal/platform/config/config.go +++ b/internal/platform/config/config.go @@ -15,8 +15,6 @@ import ( "time" ) -// Logging selects the log format, text for dev and JSON otherwise, and the -// level. type Logging struct { Env string Level slog.Level @@ -44,8 +42,7 @@ type Config struct { DrainDelay time.Duration IdempotencyTTL time.Duration DBMaxConns int32 - // DebugAddr enables the internal expvar listener when set. - DebugAddr string + DebugAddr string } type Migrate struct { diff --git a/scripts/bench/lib.sh b/scripts/bench/lib.sh index 24232ee8..991fe07c 100644 --- a/scripts/bench/lib.sh +++ b/scripts/bench/lib.sh @@ -32,7 +32,8 @@ meta() { # host_probe prints the seconds a fixed CPU loop takes in a one-CPU container: # the same work every time, so a slow spell of the host shows in the results. host_probe() { - docker run --rm --cpus 1 --entrypoint sh mtusi-messenger:lab3 -c 'time -p awk "BEGIN{for(i=0;i<5000000;i++)s+=i}"' 2>&1 | awk '/^real/ { print $2 }' + docker run --rm --cpus 1 --entrypoint sh mtusi-messenger:lab3 \ + -c 'time -p awk "BEGIN{for(i=0;i<5000000;i++)s+=i}"' 2>&1 | awk '/^real/ { print $2 }' } # sample_docker_stats appends the CPU and memory of every container until killed. diff --git a/scripts/bench/plot.py b/scripts/bench/plot.py index 33305612..cced9925 100755 --- a/scripts/bench/plot.py +++ b/scripts/bench/plot.py @@ -7,9 +7,9 @@ The raw root holds //rep/ directories. Written to the output directory: summary.csv one row per step of every run; capacity.csv the capacity of every run and the median per configuration; - control.csv the noise checks of every run: the host speed probe and the - loadgen CPU per request at the first step, flagged when more - than 25 % off the median; + control.csv the noise checks of every run, the host speed probe and the + loadgen CPU per request at the first step, each flagged when + more than 25 % off its median; -throughput.png, -p99.png, capacity.png, cpu-at-capacity.png, pool-waits.png. Capacity is the highest step that passed the criterion loadgen applies (valid, @@ -147,6 +147,7 @@ def main(root, out): capacities.append({ "series": run["series"], "config": run["config"], "rep": run["rep"], "capacity_rps": best["offered_rps"] if best else 0, "at": best, + "invalid_below": sum(1 for s in run["steps"] if not s["valid"] and (best is None or s["offered_rps"] < best["offered_rps"])), }) with open(out / "summary.csv", "w", newline="", encoding="utf-8") as f: @@ -160,27 +161,35 @@ def main(root, out): configs.append((c["series"], c["config"])) with open(out / "capacity.csv", "w", newline="", encoding="utf-8") as f: w = csv.writer(f) - w.writerow(["series", "config", "capacity_rps_per_repeat", "capacity_rps_median"]) + w.writerow(["series", "config", "capacity_rps_per_repeat", "capacity_rps_median", "invalid_steps_below_capacity"]) medians = {} for key in configs: - values = [c["capacity_rps"] for c in capacities if (c["series"], c["config"]) == key] + mine = [c for c in capacities if (c["series"], c["config"]) == key] + values = [c["capacity_rps"] for c in mine] medians[key] = statistics.median(values) - w.writerow([key[0], key[1], " ".join(f"{v:g}" for v in values), f"{medians[key]:g}"]) + w.writerow([key[0], key[1], " ".join(f"{v:g}" for v in values), f"{medians[key]:g}", + sum(c["invalid_below"] for c in mine)]) control = [] for run in runs: first = [r for r in rows if (r["series"], r["config"], r["rep"]) == (run["series"], run["config"], run["rep"])][:1] if first and first[0]["cpu_loadgen_pct"] != "": - control.append([run["series"], run["config"], run["rep"], run["meta"].get("host_probe_s", ""), first[0]["offered_rps"], - round(float(first[0]["cpu_loadgen_pct"]) / first[0]["offered_rps"] * 1000, 1)]) + probe = run["meta"].get("host_probe_s", "") + control.append([run["series"], run["config"], run["rep"], float(probe) if probe else None, + round(float(first[0]["cpu_loadgen_pct"]) * 10000 / first[0]["offered_rps"])]) if control: - median = statistics.median(c[5] for c in control) + probes = [c[3] for c in control if c[3] is not None] + probe_median = statistics.median(probes) if probes else None + cpu_median = statistics.median(c[4] for c in control) with open(out / "control.csv", "w", newline="", encoding="utf-8") as f: w = csv.writer(f) - w.writerow(["series", "config", "rep", "host_probe_s", "offered_rps", "loadgen_cpu_ms_per_1000_requests", "deviation", "flagged"]) - for c in control: - deviation = c[5] / median - 1 - w.writerow([*c, f"{deviation:+.2f}", abs(deviation) > CONTROL_TOLERANCE]) + w.writerow(["series", "config", "rep", "host_probe_s", "probe_deviation", + "loadgen_cpu_ms_per_1000_requests", "cpu_deviation", "flagged"]) + for series, config, rep, probe, cpu in control: + probe_dev = probe / probe_median - 1 if probe is not None else 0.0 + cpu_dev = cpu / cpu_median - 1 + w.writerow([series, config, rep, probe if probe is not None else "", f"{probe_dev:+.2f}", cpu, + f"{cpu_dev:+.2f}", max(abs(probe_dev), abs(cpu_dev)) > CONTROL_TOLERANCE]) def median_by_offered(series, config, field): points, valid = {}, {} diff --git a/scripts/bench/run-series.sh b/scripts/bench/run-series.sh index b25d7389..219665f0 100755 --- a/scripts/bench/run-series.sh +++ b/scripts/bench/run-series.sh @@ -62,6 +62,11 @@ run_config() { grep '^step' "$dir/loadgen.log" || true } +for conf in "$@"; do + [ -r "$conf" ] || { echo "cannot read $conf" >&2; exit 2; } + bad=$(grep -v '^#' "$conf" | awk 'NF && NF != 8') + [ -z "$bad" ] || { echo "$conf: a line needs 8 fields: $bad" >&2; exit 2; } +done docker compose --profile bench build loadgen >/dev/null default_ifs=$IFS newline=' diff --git a/tools/loadgen/main.go b/tools/loadgen/main.go index e9dd9fc0..f081d981 100644 --- a/tools/loadgen/main.go +++ b/tools/loadgen/main.go @@ -88,7 +88,7 @@ func parse(mode string, args []string) (config, error) { fs.StringVar(&cfg.Duration, "duration", "60s", "timeline: duration") fs.StringVar(&cfg.Timeout, "timeout", "2s", "how long the generator waits for a response (class client_timeout)") fs.Uint64Var(&cfg.Seed, "seed", 1, "random seed of the request mix") - fs.BoolVar(&cfg.StopOnFail, "stop-on-fail", false, "bench: end the ladder after the first step that fails the capacity criterion") + fs.BoolVar(&cfg.StopOnFail, "stop-on-fail", false, "bench: end the ladder after the first step over 1 % errors or 500 ms p99") fs.BoolVar(&cfg.Scrape, "scrape", false, "sample HAProxy statistics and replica expvar every second") fs.StringVar(&cfg.StatsURL, "stats-url", "http://lb:8404/;csv", "HAProxy statistics CSV") fs.StringVar(&cfg.APIHost, "api-host", "api", "DNS name of the API replicas") @@ -208,7 +208,7 @@ func runBench(ctx context.Context, cfg config) error { } fmt.Printf("step %6.0f rps: ok %7.1f rps, errors %5.2f%%, p99 %7.1f ms, lag p99 %5.1f ms, valid %v, passed %v\n", rate, s.OKRPS, 100*s.ErrorShare, s.P99Ms, s.LagP99Ms, s.Valid, s.Passed) - if cfg.StopOnFail && !s.Passed { + if cfg.StopOnFail && s.overloaded() { break } } diff --git a/tools/loadgen/stats.go b/tools/loadgen/stats.go index 541299f0..8f5dd8ca 100644 --- a/tools/loadgen/stats.go +++ b/tools/loadgen/stats.go @@ -121,10 +121,14 @@ func summarize(rate float64, d time.Duration, samples []sample) stepSummary { lagP99 := percentile(lags, 99) s.LagP99Ms = ms(lagP99) s.Valid = lagP99 <= maxValidLag - s.Passed = s.Valid && s.ErrorShare < maxErrorShare && percentile(ok, 99) <= sloP99 + s.Passed = s.Valid && !s.overloaded() return s } +func (s stepSummary) overloaded() bool { + return s.ErrorShare >= maxErrorShare || s.P99Ms > ms(sloP99) +} + func ms(d time.Duration) float64 { return float64(d.Microseconds()) / 1000 } diff --git a/tools/loadgen/stats_test.go b/tools/loadgen/stats_test.go index df7b6fea..f9a8bf1b 100644 --- a/tools/loadgen/stats_test.go +++ b/tools/loadgen/stats_test.go @@ -126,3 +126,23 @@ func TestStepPassesOnlyWithinTheObjective(t *testing.T) { t.Fatalf("an invalid step must fail: %+v", s) } } + +// The ladder stops when the system leaves the objective, not when the +// generator alone fell behind: a later step may still be valid and pass. +func TestOnlyTheSystemEndsTheLadder(t *testing.T) { + ok := func(latency, lag time.Duration) sample { + return sample{class: "ok", latency: latency, lag: lag} + } + late := []sample{ok(10*time.Millisecond, 50*time.Millisecond), ok(10*time.Millisecond, 50*time.Millisecond)} + if s := summarize(2, time.Second, late); s.overloaded() { + t.Fatalf("a lagging generator must not end the ladder: %+v", s) + } + slow := []sample{ok(10*time.Millisecond, time.Millisecond), ok(600*time.Millisecond, time.Millisecond)} + if s := summarize(2, time.Second, slow); !s.overloaded() { + t.Fatalf("p99 above 500 ms must end the ladder: %+v", s) + } + failing := []sample{ok(10*time.Millisecond, time.Millisecond), {class: "client_timeout", lag: time.Millisecond}} + if s := summarize(2, time.Second, failing); !s.overloaded() { + t.Fatalf("errors above 1 %% must end the ladder: %+v", s) + } +}