diff --git a/Makefile b/Makefile index 25a48ec5..f7f2119a 100644 --- a/Makefile +++ b/Makefile @@ -12,7 +12,7 @@ API_IMAGE ?= mtusi-messenger:lab3 FROM_REV ?= SCRIPTS_PY = scripts/probe.py scripts/limits.py scripts/scaling.py tools TRACK1 = evidence/track1 -SCALING = $(PYTHON) scripts/scaling.py --fresh-db --start-per-cpu 150 +SCALING = $(PYTHON) scripts/scaling.py --fresh-db PSQL = $(COMPOSE) exec -T postgres psql -U messenger -d messenger -X -q .PHONY: help check validate test load test-persistence test-failover \ @@ -124,10 +124,12 @@ track1-host: sh scripts/host-snapshot.sh $(TRACK1)/host.txt track1-a: - $(SCALING) --out $(TRACK1)/series-a --configs 1x4,2x2,4x1 --api-memory-total 2048 + $(SCALING) --out $(TRACK1)/series-a --configs 1x4,2x2,4x1 --api-memory-total 2048 \ + --start-per-cpu 150 track1-b: - $(SCALING) --out $(TRACK1)/series-b --configs 1x1,2x1,8x1 --api-memory 512 + $(SCALING) --out $(TRACK1)/series-b --configs 1x1,2x1,8x1 --api-memory 512 \ + --start-per-cpu 300 --search 1.5:9000 plots: MPLBACKEND=Agg $(UV) run --no-project --python 3.14 --with 'matplotlib==3.11.2' \ diff --git a/scripts/docker-stats.sh b/scripts/docker-stats.sh index f3c524cd..48ea8514 100755 --- a/scripts/docker-stats.sh +++ b/scripts/docker-stats.sh @@ -1,11 +1,15 @@ #!/bin/sh set -eu out=${1:?usage: docker-stats.sh } +vm=$(dirname "$out")/vm_stat.csv cd "$(dirname "$0")/.." project=$(docker compose config | sed -n 's/^name: //p') echo 'ts,name,cpu_pct,mem_usage,mem_pct,net_io,block_io,pids' > "$out" +echo 'ts,busy,total,cpus' > "$vm" while :; do + ts=$(date +%s) + docker compose exec -T valkey cat /proc/stat 2>/dev/null | awk -v ts="$ts" '/^cpu /{b=$2+$3+$4+$7+$8+$9; t=b+$5+$6} /^cpu[0-9]/{n++} END{if (t) print ts "," b "," t "," n}' >> "$vm" || true ids=$(docker ps -q --filter "label=com.docker.compose.project=$project") - [ -z "$ids" ] || docker stats --no-stream --format "$(date +%s),{{.Name}},{{.CPUPerc}},{{.MemUsage}},{{.MemPerc}},{{.NetIO}},{{.BlockIO}},{{.PIDs}}" $ids >> "$out" || true + [ -z "$ids" ] || docker stats --no-stream --format "$ts,{{.Name}},{{.CPUPerc}},{{.MemUsage}},{{.MemPerc}},{{.NetIO}},{{.BlockIO}},{{.PIDs}}" $ids >> "$out" || true sleep "${INTERVAL:-1}" done diff --git a/scripts/scaling.py b/scripts/scaling.py index a33846f7..4074e688 100755 --- a/scripts/scaling.py +++ b/scripts/scaling.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 import argparse import collections +import contextlib import csv import json import math @@ -32,7 +33,8 @@ FIELDS = ( "dropped_iterations p50_ms p95_ms p99_ms p95_post_ms p95_get_ms cpu_api cpu_postgres " "cpu_lb cpu_valkey cpu_k6 mem_api_mb mem_postgres_mb mem_k6_mb pg_backends_max " "pg_active_avg pg_active_max pg_idle_avg pg_idle_tx_avg pg_lock_avg pg_lock_max pg_lwlock_avg " - "pg_io_avg lb_scur_max lb_qcur_max lb_qtime_max_ms lb_rtime_avg_ms" + "pg_io_avg lb_scur_max lb_qcur_max lb_qtime_max_ms lb_rtime_avg_ms cpu_vm cpu_other " + "gap_s wall_s disturbed" ).split() @@ -73,6 +75,38 @@ def docker_usage(path, project, start, end): return usage +def vm_usage(path, start, end): + samples = [(int(r["busy"]), int(r["total"]), int(r["cpus"])) for r in rows(path, start, end)] + if len(samples) < 2 or samples[-1][1] == samples[0][1]: + return 0.0 + first, last = samples[0], samples[-1] + return round(100 * last[2] * (last[0] - first[0]) / (last[1] - first[1]), 1) + + +def gap(path, start, end): + ticks = sorted({float(r["ts"]) for r in rows(path, start - 5, end + 5)}) + edges = [start, *[t for t in ticks if start <= t <= end], end] + return max(b - a for a, b in zip(edges, edges[1:])) + + +def vm_counters(): + lines = compose("exec", "-T", "valkey", "cat", "/proc/stat").splitlines() + values = [int(v) for v in lines[0].split()[1:]] + cpus = sum(1 for line in lines if re.match(r"cpu\d", line)) + return sum(values) - values[3] - values[4], sum(values), cpus + + +def wait_quiet(args): + deadline, calm = time.monotonic() + args.quiet_timeout, 0 + while calm < 3 and time.monotonic() < deadline: + first = vm_counters() + time.sleep(5) + last = vm_counters() + busy = 100 * last[2] * (last[0] - first[0]) / max(1, last[1] - first[1]) + calm = calm + 1 if busy < args.max_other else 0 + print(json.dumps({"waiting_for_quiet_vm_cpu": round(busy, 1)}), flush=True) + + def pg_usage(path, start, end): ticks = collections.defaultdict(collections.Counter) for r in rows(path, start, end): @@ -192,9 +226,11 @@ def measure(folder, args, rate, env): sampler("pg-stats.sh", "activity", str(folder / "pg_activity.csv")), ] load = dict(env, RATE=str(rate), WARMUP=str(args.warmup), DURATION=str(args.duration)) + began = time.time() try: code, started = run_k6(folder / "k6", load) finally: + wall = time.time() - began for process in samplers: stop(process) subprocess.run( @@ -202,13 +238,13 @@ def measure(folder, args, rate, env): ) if started is None: raise SystemExit(f"k6 failed in setup, see {folder / 'k6/k6.txt'}") - return code, started + args.warmup, database + return code, started + args.warmup, database, wall def step(base, rate, repeat, args, env): folder = Path(args.out) / f"{base['variant']}-{base['config']}_rate{rate}_rep{repeat}" folder.mkdir(parents=True, exist_ok=True) - code, start, database = measure(folder, args, rate, env) + code, start, database, wall = measure(folder, args, rate, env) end = start + args.duration summary = folder / "k6/summary.json" metrics = k6_metrics(summary, args.duration) @@ -226,6 +262,16 @@ def step(base, rate, repeat, args, env): row.update(docker_usage(folder / "docker_stats.csv", args.project, start, end)) row.update(pg_usage(folder / "pg_activity.csv", start, end)) row.update(lb_usage(folder / "lb_stats.csv", start, end)) + row["cpu_vm"] = vm_usage(folder / "vm_stat.csv", start, end) + own = sum(row[f"cpu_{name}"] for name in ("api", "postgres", "lb", "valkey", "k6")) + row["cpu_other"] = round(row["cpu_vm"] - own, 1) + row["gap_s"] = round(gap(folder / "docker_stats.csv", start, end), 1) + row["wall_s"] = round(wall, 1) + row["disturbed"] = ( + row["cpu_other"] > args.max_other + or row["gap_s"] > args.max_gap + or wall > args.warmup + args.duration + args.max_overhead + ) print(json.dumps({k: row[k] for k in FIELDS}, ensure_ascii=False), flush=True) return row @@ -269,6 +315,11 @@ def arguments(): parser.add_argument("--slo-p95-ms", type=float, default=500.0) parser.add_argument("--max-error-ratio", type=float, default=0.01) parser.add_argument("--keep", action="store_true", help="leave the last layout running") + parser.add_argument("--max-other", type=float, default=100.0, help="foreign VM CPU, %% core") + parser.add_argument("--max-gap", type=float, default=10.0, help="sampling gap, s") + parser.add_argument("--max-overhead", type=float, default=90.0, help="k6 run beyond load, s") + parser.add_argument("--retries", type=int, default=3, help="reruns of a disturbed step") + parser.add_argument("--quiet-timeout", type=float, default=1200.0) args = parser.parse_args() args.variants = args.variants or ["base"] if len(args.variants) > 1 and not args.rates: @@ -282,8 +333,14 @@ def block(args, output, config, variant, repeats): for rate in rates(args, base): results = [] for repeat in repeats: - results.append(step(base, rate, repeat, args, current)) - time.sleep(args.pause) + for _ in range(args.retries + 1): + row = step(base, rate, repeat, args, current) + time.sleep(args.pause) + if not row["disturbed"]: + break + args.rejected.writerow(row) + wait_quiet(args) + results.append(row) output.writerows(results) streak = streak + 1 if not args.rates and all(r["saturated"] for r in results) else 0 if streak == 2: @@ -303,19 +360,24 @@ def series(args, output): block(args, output, config, variant, [repeat]) +@contextlib.contextmanager +def table(path): + fresh = not path.exists() + with path.open("a", newline="", buffering=1) as handle: + writer = csv.DictWriter(handle, fieldnames=FIELDS, extrasaction="ignore") + if fresh: + writer.writeheader() + yield writer + + def main(): args = arguments() out = Path(args.out) out.mkdir(parents=True, exist_ok=True) args.project = json.loads(compose("config", "--format", "json"))["name"] args.base_env = dict(os.environ, **(PINNED if args.pin else {})) - summary = out / "summary.csv" - fresh = not summary.exists() try: - with summary.open("a", newline="", buffering=1) as handle: - writer = csv.DictWriter(handle, fieldnames=FIELDS, extrasaction="ignore") - if fresh: - writer.writeheader() + with table(out / "summary.csv") as writer, table(out / "disturbed.csv") as args.rejected: series(args, writer) finally: if not args.keep: diff --git a/tools/plot_track1.py b/tools/plot_track1.py index 2c8c4c46..24691a6f 100644 --- a/tools/plot_track1.py +++ b/tools/plot_track1.py @@ -9,6 +9,7 @@ import matplotlib.pyplot as plt ORDER = ("1x1", "2x1", "4x1", "8x1", "2x2", "1x4") PALETTE = ("#2a78d6", "#eb6834", "#1baf7a", "#eda100", "#e87ba4", "#008300", "#6250d6", "#e34948") MARKERS = ("o", "s", "^", "D", "v", "P", "X", "*") +USL_POINTS = 3 COMPONENTS = (("api", "API"), ("postgres", "PostgreSQL"), ("lb", "HAProxy"), ("k6", "k6")) INK, MUTED, GRID, SURFACE = "#0b0b0b", "#52514e", "#e4e3df", "#fcfcfb" SUMMARY = ( @@ -170,6 +171,17 @@ def cpu(series, keys, groups, out): plt.close(figure) +def usl(points): + base = points[0][1] / points[0][0] + pairs = [(n - 1, n * (n - 1), n * base / x - 1) for n, x in points] + a, b, c = (sum(p[i] * p[j] for p in pairs) for i, j in ((0, 0), (0, 1), (1, 1))) + d, e = (sum(p[i] * p[2] for p in pairs) for i in (0, 1)) + det = a * c - b * b + sigma, kappa = (d * c - b * e) / det, (a * e - b * d) / det + peak = ((1 - sigma) / kappa) ** 0.5 if kappa > 0 else None + return base, sigma, kappa, peak + + def scaling(rows, out): by = {(r["series"], r["config"]): r for r in rows if r["variant"] == "base"} growth = [ @@ -183,6 +195,15 @@ def scaling(rows, out): unit = growth[0]["peak_ok_rps"] / cpus[0] axis.plot(cpus, [unit * c for c in cpus], color=MUTED, linestyle="--", linewidth=1) axis.annotate("линейный рост", (cpus[-1], unit * cpus[-1]), color=MUTED, fontsize=9) + fit = None + if len(growth) >= USL_POINTS: + fit = usl([(c, r["peak_ok_rps"]) for c, r in zip(cpus, growth, strict=True)]) + base, sigma, kappa, _ = fit + grid = [1 + i * (cpus[-1] - 1) / 60 for i in range(61)] + model = [base * n / (1 + sigma * (n - 1) + kappa * n * (n - 1)) for n in grid] + axis.plot( + grid, model, color=PALETTE[6], linewidth=1.5, label=f"USL: σ={sigma:.3f}, κ={kappa:.4f}" + ) axis.errorbar( cpus, [r["peak_ok_rps"] for r in growth], @@ -210,6 +231,7 @@ def scaling(rows, out): figure.tight_layout() figure.savefig(out / "scaling.png", dpi=150) plt.close(figure) + return fit def utilisation(rows, out): @@ -251,7 +273,7 @@ def utilisation(rows, out): plt.close(figure) -def write(rows, out): +def write(rows, out, fit): with (out / "capacity.csv").open("w", newline="") as handle: writer = csv.DictWriter(handle, fieldnames=SUMMARY, extrasaction="ignore") writer.writeheader() @@ -269,6 +291,12 @@ def write(rows, out): f"{r['cpu_api']:.0f} / {r['cpu_api_limit']} | {r['cpu_postgres']:.0f} | " f"{r['cpu_lb']:.0f} | {r['pg_lock_avg']:.1f} / {r['pool_total']} |" ) + if fit: + _, sigma, kappa, peak = fit + best = f"{peak:.1f}" if peak else "—" + lines_.append( + f"\nUSL по серии B: σ = {sigma:.4f}, κ = {kappa:.5f}, пик при N* = {best} CPU" + ) (out / "capacity.md").write_text("\n".join(lines_) + "\n") print("\n".join(lines_)) @@ -288,9 +316,9 @@ def main(): chosen = [k for k in keys if k[0] == series] throughput(series, chosen, groups, out) cpu(series, chosen, groups, out) - scaling(rows, out) + fit = scaling(rows, out) utilisation(rows, out) - write(rows, out) + write(rows, out, fit) if __name__ == "__main__":