From 9cfe15ef1af8f8b32a7bc4a965602005fe404430 Mon Sep 17 00:00:00 2001 From: kk10 Date: Tue, 20 Oct 2026 13:18:07 +0300 Subject: [PATCH] feat(bench): add load test flags, bench matrix and plot scripts --- scripts/bench.sh | 93 +++++++++++++++++++++++++ scripts/plot.py | 108 ++++++++++++++++++++++++++++++ student-tests/load_action_test.go | 4 ++ student-tests/load_test.go | 25 +++++++ student.mk | 6 +- 5 files changed, 234 insertions(+), 2 deletions(-) create mode 100644 scripts/bench.sh create mode 100644 scripts/plot.py diff --git a/scripts/bench.sh b/scripts/bench.sh new file mode 100644 index 0000000..caadb96 --- /dev/null +++ b/scripts/bench.sh @@ -0,0 +1,93 @@ +#!/bin/sh +# Benchmark matrix: configs x load steps x repeats. +# Run: sh scripts/bench.sh +# Quick run: RUNS=1 RATES="10 40" DURATION=10s CONFIGS="2:1.0:256M" sh scripts/bench.sh +set -eu + +RUNS=${RUNS:-3} # repeats per point +WORKERS=${WORKERS:-20} # target rate = WORKERS * rate +RATES=${RATES:-"20 40 80"} # req/s per worker +DURATION=${DURATION:-20s} # per run +PAUSE=${PAUSE:-5} # seconds between runs +WARMUP=${WARMUP:-5s} # per config, not recorded +OUT_DIR=${OUT_DIR:-results} + +# Config format: replicas:CPU per replica:RAM per replica +# Series 1: fixed API budget 2.0 CPU / 512M. Series 2: 2 replicas, growing budget. +CONFIGS=${CONFIGS:-"1:2.0:512M 2:1.0:256M 4:0.5:128M 2:0.5:256M 2:2.0:256M"} + +mkdir -p "$OUT_DIR/stats" "$OUT_DIR/logs" +CSV="$(pwd)/$OUT_DIR/bench.csv" # absolute: go test runs in student-tests/ +if [ ! -f "$CSV" ]; then + echo "label,workers,rate,target,ok,non201,errors,rps,p50_ms,p95_ms,p99_ms" > "$CSV" +fi + +SAMPLER_PID="" +stop_sampler() { + if [ -n "$SAMPLER_PID" ]; then + kill "$SAMPLER_PID" 2>/dev/null || true + wait "$SAMPLER_PID" 2>/dev/null || true + SAMPLER_PID="" + fi +} +cleanup() { + stop_sampler + # restore default stand + env -u API_REPLICAS -u API_CPUS -u API_MEM -u LB_CPUS docker compose --progress quiet up -d --wait || true +} +trap cleanup EXIT INT TERM + +start_sampler() { # $1: output file + ( + while :; do + ts=$(date +%s) + docker stats --no-stream --format '{{.Name}},{{.CPUPerc}},{{.MemUsage}}' | + while read -r line; do echo "$ts,$line"; done >> "$1" + sleep 1 + done + ) & + SAMPLER_PID=$! +} + +run_config() { # N CPUS MEM + N=$1; CPUS=$2; MEM=$3 + label="n${N}_cpu${CPUS}" + echo "== $label (API: ${N} x ${CPUS} CPU, ${MEM}) ==" + + API_REPLICAS=$N API_CPUS=$CPUS API_MEM=$MEM docker compose --progress quiet up -d --wait + sleep 3 # let nginx refresh DNS (valid=2s) + docker compose ps -a > "$OUT_DIR/logs/ps_${label}.txt" # proof of replica count + docker inspect "$(docker compose ps -q load_balancer)" --format '{{.HostConfig.NanoCpus}}' > "$OUT_DIR/logs/lb_nanocpus_${label}.txt" + + # warmup + make load-go LOAD_WORKERS="$WORKERS" LOAD_RATE=10 LOAD_DURATION="$WARMUP" > /dev/null 2>&1 || true + + for rate in $RATES; do + run=1 + while [ "$run" -le "$RUNS" ]; do + tag="${label}_r${rate}_run${run}" + start_sampler "$OUT_DIR/stats/${tag}.csv" + + make load-go \ + LOAD_WORKERS="$WORKERS" LOAD_RATE="$rate" LOAD_DURATION="$DURATION" \ + LOAD_CSV="$CSV" LOAD_LABEL="$label" \ + > "$OUT_DIR/logs/${tag}.log" 2>&1 || echo " run $tag failed" + + stop_sampler + sleep "$PAUSE" # let the replica recover after overload + until curl -fs "${BASE_URL:-http://localhost:8080}/health/ready" > /dev/null; do sleep 1; done + echo " rate=$rate run=$run: $(grep -E 'successful RPS|latency p95' "$OUT_DIR/logs/${tag}.log" | sed 's/.*load_test.go:[0-9]*: //' | tr '\n' ' ')" + run=$((run + 1)) + done + done +} + +for cfg in $CONFIGS; do + N=${cfg%%:*} + rest=${cfg#*:} + CPUS=${rest%%:*} + MEM=${rest#*:} + run_config "$N" "$CPUS" "$MEM" +done + +echo "Done: $CSV, container stats in $OUT_DIR/stats, logs in $OUT_DIR/logs" diff --git a/scripts/plot.py b/scripts/plot.py new file mode 100644 index 0000000..133bb51 --- /dev/null +++ b/scripts/plot.py @@ -0,0 +1,108 @@ +import csv +import statistics as st +from pathlib import Path + +import matplotlib + +matplotlib.use("Agg") +import matplotlib.pyplot as plt + +ROOT = Path("evidence/track1") +OUT = ROOT / "plots" +CONFIGS = ["n1_cpu2.0", "n2_cpu1.0", "n4_cpu0.5"] +NAMES = {"n1_cpu2.0": "1 реплика", "n2_cpu1.0": "2 реплики", "n4_cpu0.5": "4 реплики"} +QUOTA = {"postgres": 100, "redis": 25, "load_balancer": 100, "api": 50} +SVC_NAMES = {"postgres": "PostgreSQL", "redis": "Redis", "load_balancer": "Nginx", "api": "API (реплика)"} + + +def load(d): + """(label, rate) -> {"target": int, "rps": [...], "p95": [...]}""" + data = {} + with open(ROOT / d / "bench.csv", newline="") as f: + for r in csv.DictReader(f): + e = data.setdefault((r["label"], int(r["rate"])), + {"target": int(r["target"]), "rps": [], "p95": []}) + e["rps"].append(float(r["rps"])) + e["p95"].append(float(r["p95_ms"])) + return data + + +def scaling(data): + """RPS и p95 от нагрузки, линия на конфигурацию.""" + fig, axes = plt.subplots(1, 2, figsize=(11, 4.2)) + for ax, field, ylabel in zip(axes, ("rps", "p95"), ("успешные RPS", "p95, мс")): + for lb in CONFIGS: + pts = sorted((e["target"], e[field]) for (l, _), e in data.items() if l == lb) + x = [t for t, _ in pts] + ax.plot(x, [st.median(v) for _, v in pts], marker="o", label=NAMES[lb]) + ax.fill_between(x, [min(v) for _, v in pts], [max(v) for _, v in pts], alpha=0.15) + ax.set(xlabel="целевая нагрузка, req/s", ylabel=f"{ylabel} (медиана, min–max)") + ax.grid(alpha=0.3) + axes[0].legend() + fig.suptitle("1/2/4 реплики при бюджете API 2 CPU (Nginx 0,5 CPU)") + fig.tight_layout() + fig.savefig(OUT / "scaling.png", dpi=130) + plt.close(fig) + + +def before_after(before, after, rate=80): + """Nginx 0,5 CPU против 1,0 CPU на ступени rate.""" + fig, axes = plt.subplots(1, 2, figsize=(11, 4.2)) + w = 0.38 + for ax, field, ylabel in zip(axes, ("rps", "p95"), ("успешные RPS", "p95, мс")): + for i, (data, title) in enumerate(((before, "Nginx 0,5 CPU"), (after, "Nginx 1,0 CPU"))): + vals = [data[(lb, rate)][field] for lb in CONFIGS] + med = [st.median(v) for v in vals] + err = [[m - min(v) for m, v in zip(med, vals)], [max(v) - m for m, v in zip(med, vals)]] + ax.bar([j + (i - 0.5) * w for j in range(len(CONFIGS))], med, w, + yerr=err, capsize=3, label=title) + ax.set_xticks(range(len(CONFIGS))) + ax.set_xticklabels([NAMES[c] for c in CONFIGS]) + ax.set_ylabel(f"{ylabel} (медиана, min–max)") + ax.grid(axis="y", alpha=0.3) + axes[0].legend() + fig.suptitle(f"Улучшение: квота Nginx 0,5 → 1,0 CPU (ступень {rate} req/s на воркера)") + fig.tight_layout() + fig.savefig(OUT / "lb_before_after.png", dpi=130) + plt.close(fig) + + +def cpu_at_limit(d="generator_check", label="n4_cpu0.5", rate=80): + """CPU контейнеров на пределе 4 реплик: медиана замеров docker stats.""" + samples = {s: [] for s in QUOTA} + for f in sorted((ROOT / d / "stats").glob(f"{label}_r{rate}_run*.csv")): + with open(f, newline="") as fh: + for row in csv.reader(fh): + if len(row) < 3: + continue + for svc in QUOTA: + if svc in row[1]: + samples[svc].append(float(row[2].rstrip("%"))) + break + svcs = [s for s in QUOTA if samples[s]] + med = [st.median(samples[s]) for s in svcs] + names = [SVC_NAMES[s] for s in svcs] + fig, ax = plt.subplots(figsize=(7.5, 4.2)) + bars = ax.bar(names, med, 0.5) + ax.bar_label(bars, fmt="%.0f%%") + ax.plot(names, [QUOTA[s] for s in svcs], "k_", markersize=40, mew=2, label="квота CPU") + ax.set_ylabel("CPU, % одного ядра (медиана замеров)") + ax.set_title(f"CPU контейнеров на пределе: {NAMES[label]}, 40 воркеров, ступень {rate}") + ax.legend() + ax.grid(axis="y", alpha=0.3) + fig.tight_layout() + fig.savefig(OUT / "cpu_at_limit.png", dpi=130) + plt.close(fig) + + +def main(): + OUT.mkdir(parents=True, exist_ok=True) + before, after = load("before"), load("after") + scaling(before) + before_after(before, after) + cpu_at_limit() + print(f"Графики записаны в {OUT}") + + +if __name__ == "__main__": + main() diff --git a/student-tests/load_action_test.go b/student-tests/load_action_test.go index e1094b8..1e38747 100644 --- a/student-tests/load_action_test.go +++ b/student-tests/load_action_test.go @@ -45,6 +45,10 @@ func TestLoadWithAction(t *testing.T) { t.Skip("-action is not set") } + if *actionAt+10*time.Second > *duration { + t.Fatalf("-duration (%s) must exceed -action-at (%s) by at least 10s", *duration, *actionAt) + } + user1 := createUser(t) user2 := createUser(t) chat := createChat(t, user1, user1.ID, user2.ID) diff --git a/student-tests/load_test.go b/student-tests/load_test.go index a0f5b0f..a4a0902 100644 --- a/student-tests/load_test.go +++ b/student-tests/load_test.go @@ -4,8 +4,10 @@ package studenttests_test import ( "flag" + "fmt" "io" "net/http" + "os" "sort" "sync" "sync/atomic" @@ -17,6 +19,8 @@ var ( workers = flag.Int("workers", 20, "number of concurrent workers") duration = flag.Duration("duration", 5*time.Second, "test duration") rate = flag.Int("rate", 20, "maximum requests per second per worker") + csvPath = flag.String("csv", "", "append result row to this CSV file") + label = flag.String("label", "", "configuration label for CSV row") ) func TestLoad(t *testing.T) { @@ -129,6 +133,27 @@ func TestLoad(t *testing.T) { t.Logf("successful latency p95: %s", percentile(latencies, 0.95)) t.Logf("successful latency p99: %s", percentile(latencies, 0.99)) } + + if *csvPath != "" { + var p50, p95, p99 time.Duration + if len(latencies) > 0 { + p50 = percentile(latencies, 0.50) + p95 = percentile(latencies, 0.95) + p99 = percentile(latencies, 0.99) + } + f, err := os.OpenFile(*csvPath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o644) + if err != nil { + t.Fatalf("open csv: %v", err) + } + defer f.Close() + fmt.Fprintf(f, "%s,%d,%d,%d,%d,%d,%d,%.2f,%.1f,%.1f,%.1f\n", + *label, *workers, *rate, *workers**rate, + successful, total-successful-requestErrors, requestErrors, + float64(successful)/elapsed.Seconds(), + float64(p50.Microseconds())/1000, + float64(p95.Microseconds())/1000, + float64(p99.Microseconds())/1000) + } } func percentile(values []time.Duration, p float64) time.Duration { diff --git a/student.mk b/student.mk index 1808594..510a224 100644 --- a/student.mk +++ b/student.mk @@ -1,6 +1,8 @@ LOAD_ARGS = $(if $(LOAD_WORKERS),-workers=$(LOAD_WORKERS),) \ $(if $(LOAD_DURATION),-duration=$(LOAD_DURATION),) \ - $(if $(LOAD_RATE),-rate=$(LOAD_RATE),) + $(if $(LOAD_RATE),-rate=$(LOAD_RATE),) \ + $(if $(LOAD_CSV),-csv=$(LOAD_CSV),) \ + $(if $(LOAD_LABEL),-label=$(LOAD_LABEL),) REPLICA ?= 1 ACTION ?= stop-one.sh ACTION_AT ?= 10s @@ -20,7 +22,7 @@ race-go: go test -tags=race ./student-tests/... -race -v -args -base-url=$(BASE_URL) load-go: - go test -tags=load ./student-tests/... -v -args -base-url=$(BASE_URL) $(LOAD_ARGS) + go test -count=1 -tags=load ./student-tests/... -v -args -base-url=$(BASE_URL) $(LOAD_ARGS) failure-go: go test -tags=failure ./student-tests/... -v -args -base-url=$(BASE_URL)