forked from mtusi2026/mtusi2026lab3
feat(bench): add load test flags, bench matrix and plot scripts
This commit is contained in:
@@ -0,0 +1,93 @@
|
||||
#!/bin/sh
|
||||
# Benchmark matrix: configs x load steps x repeats.
|
||||
# Run: sh scripts/bench.sh
|
||||
# Quick run: RUNS=1 RATES="10 40" DURATION=10s CONFIGS="2:1.0:256M" sh scripts/bench.sh
|
||||
set -eu
|
||||
|
||||
RUNS=${RUNS:-3} # repeats per point
|
||||
WORKERS=${WORKERS:-20} # target rate = WORKERS * rate
|
||||
RATES=${RATES:-"20 40 80"} # req/s per worker
|
||||
DURATION=${DURATION:-20s} # per run
|
||||
PAUSE=${PAUSE:-5} # seconds between runs
|
||||
WARMUP=${WARMUP:-5s} # per config, not recorded
|
||||
OUT_DIR=${OUT_DIR:-results}
|
||||
|
||||
# Config format: replicas:CPU per replica:RAM per replica
|
||||
# Series 1: fixed API budget 2.0 CPU / 512M. Series 2: 2 replicas, growing budget.
|
||||
CONFIGS=${CONFIGS:-"1:2.0:512M 2:1.0:256M 4:0.5:128M 2:0.5:256M 2:2.0:256M"}
|
||||
|
||||
mkdir -p "$OUT_DIR/stats" "$OUT_DIR/logs"
|
||||
CSV="$(pwd)/$OUT_DIR/bench.csv" # absolute: go test runs in student-tests/
|
||||
if [ ! -f "$CSV" ]; then
|
||||
echo "label,workers,rate,target,ok,non201,errors,rps,p50_ms,p95_ms,p99_ms" > "$CSV"
|
||||
fi
|
||||
|
||||
SAMPLER_PID=""
|
||||
stop_sampler() {
|
||||
if [ -n "$SAMPLER_PID" ]; then
|
||||
kill "$SAMPLER_PID" 2>/dev/null || true
|
||||
wait "$SAMPLER_PID" 2>/dev/null || true
|
||||
SAMPLER_PID=""
|
||||
fi
|
||||
}
|
||||
cleanup() {
|
||||
stop_sampler
|
||||
# restore default stand
|
||||
env -u API_REPLICAS -u API_CPUS -u API_MEM -u LB_CPUS docker compose --progress quiet up -d --wait || true
|
||||
}
|
||||
trap cleanup EXIT INT TERM
|
||||
|
||||
start_sampler() { # $1: output file
|
||||
(
|
||||
while :; do
|
||||
ts=$(date +%s)
|
||||
docker stats --no-stream --format '{{.Name}},{{.CPUPerc}},{{.MemUsage}}' |
|
||||
while read -r line; do echo "$ts,$line"; done >> "$1"
|
||||
sleep 1
|
||||
done
|
||||
) &
|
||||
SAMPLER_PID=$!
|
||||
}
|
||||
|
||||
run_config() { # N CPUS MEM
|
||||
N=$1; CPUS=$2; MEM=$3
|
||||
label="n${N}_cpu${CPUS}"
|
||||
echo "== $label (API: ${N} x ${CPUS} CPU, ${MEM}) =="
|
||||
|
||||
API_REPLICAS=$N API_CPUS=$CPUS API_MEM=$MEM docker compose --progress quiet up -d --wait
|
||||
sleep 3 # let nginx refresh DNS (valid=2s)
|
||||
docker compose ps -a > "$OUT_DIR/logs/ps_${label}.txt" # proof of replica count
|
||||
docker inspect "$(docker compose ps -q load_balancer)" --format '{{.HostConfig.NanoCpus}}' > "$OUT_DIR/logs/lb_nanocpus_${label}.txt"
|
||||
|
||||
# warmup
|
||||
make load-go LOAD_WORKERS="$WORKERS" LOAD_RATE=10 LOAD_DURATION="$WARMUP" > /dev/null 2>&1 || true
|
||||
|
||||
for rate in $RATES; do
|
||||
run=1
|
||||
while [ "$run" -le "$RUNS" ]; do
|
||||
tag="${label}_r${rate}_run${run}"
|
||||
start_sampler "$OUT_DIR/stats/${tag}.csv"
|
||||
|
||||
make load-go \
|
||||
LOAD_WORKERS="$WORKERS" LOAD_RATE="$rate" LOAD_DURATION="$DURATION" \
|
||||
LOAD_CSV="$CSV" LOAD_LABEL="$label" \
|
||||
> "$OUT_DIR/logs/${tag}.log" 2>&1 || echo " run $tag failed"
|
||||
|
||||
stop_sampler
|
||||
sleep "$PAUSE" # let the replica recover after overload
|
||||
until curl -fs "${BASE_URL:-http://localhost:8080}/health/ready" > /dev/null; do sleep 1; done
|
||||
echo " rate=$rate run=$run: $(grep -E 'successful RPS|latency p95' "$OUT_DIR/logs/${tag}.log" | sed 's/.*load_test.go:[0-9]*: //' | tr '\n' ' ')"
|
||||
run=$((run + 1))
|
||||
done
|
||||
done
|
||||
}
|
||||
|
||||
for cfg in $CONFIGS; do
|
||||
N=${cfg%%:*}
|
||||
rest=${cfg#*:}
|
||||
CPUS=${rest%%:*}
|
||||
MEM=${rest#*:}
|
||||
run_config "$N" "$CPUS" "$MEM"
|
||||
done
|
||||
|
||||
echo "Done: $CSV, container stats in $OUT_DIR/stats, logs in $OUT_DIR/logs"
|
||||
+108
@@ -0,0 +1,108 @@
|
||||
import csv
|
||||
import statistics as st
|
||||
from pathlib import Path
|
||||
|
||||
import matplotlib
|
||||
|
||||
matplotlib.use("Agg")
|
||||
import matplotlib.pyplot as plt
|
||||
|
||||
ROOT = Path("evidence/track1")
|
||||
OUT = ROOT / "plots"
|
||||
CONFIGS = ["n1_cpu2.0", "n2_cpu1.0", "n4_cpu0.5"]
|
||||
NAMES = {"n1_cpu2.0": "1 реплика", "n2_cpu1.0": "2 реплики", "n4_cpu0.5": "4 реплики"}
|
||||
QUOTA = {"postgres": 100, "redis": 25, "load_balancer": 100, "api": 50}
|
||||
SVC_NAMES = {"postgres": "PostgreSQL", "redis": "Redis", "load_balancer": "Nginx", "api": "API (реплика)"}
|
||||
|
||||
|
||||
def load(d):
|
||||
"""(label, rate) -> {"target": int, "rps": [...], "p95": [...]}"""
|
||||
data = {}
|
||||
with open(ROOT / d / "bench.csv", newline="") as f:
|
||||
for r in csv.DictReader(f):
|
||||
e = data.setdefault((r["label"], int(r["rate"])),
|
||||
{"target": int(r["target"]), "rps": [], "p95": []})
|
||||
e["rps"].append(float(r["rps"]))
|
||||
e["p95"].append(float(r["p95_ms"]))
|
||||
return data
|
||||
|
||||
|
||||
def scaling(data):
|
||||
"""RPS и p95 от нагрузки, линия на конфигурацию."""
|
||||
fig, axes = plt.subplots(1, 2, figsize=(11, 4.2))
|
||||
for ax, field, ylabel in zip(axes, ("rps", "p95"), ("успешные RPS", "p95, мс")):
|
||||
for lb in CONFIGS:
|
||||
pts = sorted((e["target"], e[field]) for (l, _), e in data.items() if l == lb)
|
||||
x = [t for t, _ in pts]
|
||||
ax.plot(x, [st.median(v) for _, v in pts], marker="o", label=NAMES[lb])
|
||||
ax.fill_between(x, [min(v) for _, v in pts], [max(v) for _, v in pts], alpha=0.15)
|
||||
ax.set(xlabel="целевая нагрузка, req/s", ylabel=f"{ylabel} (медиана, min–max)")
|
||||
ax.grid(alpha=0.3)
|
||||
axes[0].legend()
|
||||
fig.suptitle("1/2/4 реплики при бюджете API 2 CPU (Nginx 0,5 CPU)")
|
||||
fig.tight_layout()
|
||||
fig.savefig(OUT / "scaling.png", dpi=130)
|
||||
plt.close(fig)
|
||||
|
||||
|
||||
def before_after(before, after, rate=80):
|
||||
"""Nginx 0,5 CPU против 1,0 CPU на ступени rate."""
|
||||
fig, axes = plt.subplots(1, 2, figsize=(11, 4.2))
|
||||
w = 0.38
|
||||
for ax, field, ylabel in zip(axes, ("rps", "p95"), ("успешные RPS", "p95, мс")):
|
||||
for i, (data, title) in enumerate(((before, "Nginx 0,5 CPU"), (after, "Nginx 1,0 CPU"))):
|
||||
vals = [data[(lb, rate)][field] for lb in CONFIGS]
|
||||
med = [st.median(v) for v in vals]
|
||||
err = [[m - min(v) for m, v in zip(med, vals)], [max(v) - m for m, v in zip(med, vals)]]
|
||||
ax.bar([j + (i - 0.5) * w for j in range(len(CONFIGS))], med, w,
|
||||
yerr=err, capsize=3, label=title)
|
||||
ax.set_xticks(range(len(CONFIGS)))
|
||||
ax.set_xticklabels([NAMES[c] for c in CONFIGS])
|
||||
ax.set_ylabel(f"{ylabel} (медиана, min–max)")
|
||||
ax.grid(axis="y", alpha=0.3)
|
||||
axes[0].legend()
|
||||
fig.suptitle(f"Улучшение: квота Nginx 0,5 → 1,0 CPU (ступень {rate} req/s на воркера)")
|
||||
fig.tight_layout()
|
||||
fig.savefig(OUT / "lb_before_after.png", dpi=130)
|
||||
plt.close(fig)
|
||||
|
||||
|
||||
def cpu_at_limit(d="generator_check", label="n4_cpu0.5", rate=80):
|
||||
"""CPU контейнеров на пределе 4 реплик: медиана замеров docker stats."""
|
||||
samples = {s: [] for s in QUOTA}
|
||||
for f in sorted((ROOT / d / "stats").glob(f"{label}_r{rate}_run*.csv")):
|
||||
with open(f, newline="") as fh:
|
||||
for row in csv.reader(fh):
|
||||
if len(row) < 3:
|
||||
continue
|
||||
for svc in QUOTA:
|
||||
if svc in row[1]:
|
||||
samples[svc].append(float(row[2].rstrip("%")))
|
||||
break
|
||||
svcs = [s for s in QUOTA if samples[s]]
|
||||
med = [st.median(samples[s]) for s in svcs]
|
||||
names = [SVC_NAMES[s] for s in svcs]
|
||||
fig, ax = plt.subplots(figsize=(7.5, 4.2))
|
||||
bars = ax.bar(names, med, 0.5)
|
||||
ax.bar_label(bars, fmt="%.0f%%")
|
||||
ax.plot(names, [QUOTA[s] for s in svcs], "k_", markersize=40, mew=2, label="квота CPU")
|
||||
ax.set_ylabel("CPU, % одного ядра (медиана замеров)")
|
||||
ax.set_title(f"CPU контейнеров на пределе: {NAMES[label]}, 40 воркеров, ступень {rate}")
|
||||
ax.legend()
|
||||
ax.grid(axis="y", alpha=0.3)
|
||||
fig.tight_layout()
|
||||
fig.savefig(OUT / "cpu_at_limit.png", dpi=130)
|
||||
plt.close(fig)
|
||||
|
||||
|
||||
def main():
|
||||
OUT.mkdir(parents=True, exist_ok=True)
|
||||
before, after = load("before"), load("after")
|
||||
scaling(before)
|
||||
before_after(before, after)
|
||||
cpu_at_limit()
|
||||
print(f"Графики записаны в {OUT}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -45,6 +45,10 @@ func TestLoadWithAction(t *testing.T) {
|
||||
t.Skip("-action is not set")
|
||||
}
|
||||
|
||||
if *actionAt+10*time.Second > *duration {
|
||||
t.Fatalf("-duration (%s) must exceed -action-at (%s) by at least 10s", *duration, *actionAt)
|
||||
}
|
||||
|
||||
user1 := createUser(t)
|
||||
user2 := createUser(t)
|
||||
chat := createChat(t, user1, user1.ID, user2.ID)
|
||||
|
||||
@@ -4,8 +4,10 @@ package studenttests_test
|
||||
|
||||
import (
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"sort"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
@@ -17,6 +19,8 @@ var (
|
||||
workers = flag.Int("workers", 20, "number of concurrent workers")
|
||||
duration = flag.Duration("duration", 5*time.Second, "test duration")
|
||||
rate = flag.Int("rate", 20, "maximum requests per second per worker")
|
||||
csvPath = flag.String("csv", "", "append result row to this CSV file")
|
||||
label = flag.String("label", "", "configuration label for CSV row")
|
||||
)
|
||||
|
||||
func TestLoad(t *testing.T) {
|
||||
@@ -129,6 +133,27 @@ func TestLoad(t *testing.T) {
|
||||
t.Logf("successful latency p95: %s", percentile(latencies, 0.95))
|
||||
t.Logf("successful latency p99: %s", percentile(latencies, 0.99))
|
||||
}
|
||||
|
||||
if *csvPath != "" {
|
||||
var p50, p95, p99 time.Duration
|
||||
if len(latencies) > 0 {
|
||||
p50 = percentile(latencies, 0.50)
|
||||
p95 = percentile(latencies, 0.95)
|
||||
p99 = percentile(latencies, 0.99)
|
||||
}
|
||||
f, err := os.OpenFile(*csvPath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o644)
|
||||
if err != nil {
|
||||
t.Fatalf("open csv: %v", err)
|
||||
}
|
||||
defer f.Close()
|
||||
fmt.Fprintf(f, "%s,%d,%d,%d,%d,%d,%d,%.2f,%.1f,%.1f,%.1f\n",
|
||||
*label, *workers, *rate, *workers**rate,
|
||||
successful, total-successful-requestErrors, requestErrors,
|
||||
float64(successful)/elapsed.Seconds(),
|
||||
float64(p50.Microseconds())/1000,
|
||||
float64(p95.Microseconds())/1000,
|
||||
float64(p99.Microseconds())/1000)
|
||||
}
|
||||
}
|
||||
|
||||
func percentile(values []time.Duration, p float64) time.Duration {
|
||||
|
||||
+4
-2
@@ -1,6 +1,8 @@
|
||||
LOAD_ARGS = $(if $(LOAD_WORKERS),-workers=$(LOAD_WORKERS),) \
|
||||
$(if $(LOAD_DURATION),-duration=$(LOAD_DURATION),) \
|
||||
$(if $(LOAD_RATE),-rate=$(LOAD_RATE),)
|
||||
$(if $(LOAD_RATE),-rate=$(LOAD_RATE),) \
|
||||
$(if $(LOAD_CSV),-csv=$(LOAD_CSV),) \
|
||||
$(if $(LOAD_LABEL),-label=$(LOAD_LABEL),)
|
||||
REPLICA ?= 1
|
||||
ACTION ?= stop-one.sh
|
||||
ACTION_AT ?= 10s
|
||||
@@ -20,7 +22,7 @@ race-go:
|
||||
go test -tags=race ./student-tests/... -race -v -args -base-url=$(BASE_URL)
|
||||
|
||||
load-go:
|
||||
go test -tags=load ./student-tests/... -v -args -base-url=$(BASE_URL) $(LOAD_ARGS)
|
||||
go test -count=1 -tags=load ./student-tests/... -v -args -base-url=$(BASE_URL) $(LOAD_ARGS)
|
||||
|
||||
failure-go:
|
||||
go test -tags=failure ./student-tests/... -v -args -base-url=$(BASE_URL)
|
||||
|
||||
Reference in New Issue
Block a user