#!/usr/bin/env python3 """Tables and charts of track 1 from the raw results of run-series.sh. python scripts/bench/plot.py The raw root holds //rep/ directories. Written to the output directory: summary.csv one row per step of every run; capacity.csv the capacity of every run and the median per configuration; control.csv the noise checks of every run, the host speed probe and the loadgen CPU per request at the first step, each flagged when more than 25 % off its median; cost.csv CPU milliseconds per request of each component at the COST_RATES steps (valid, error-free, unflagged runs only), and the rate one database CPU would serve at that cost; -throughput.png, -p99.png, capacity.png, cpu-at-capacity.png, pool-waits.png, cost-per-request.png. Capacity is the highest step that passed the criterion loadgen applies (valid, under 1 % errors, p99 of successful requests within 500 ms). Charts show the median of the repeats; a hollow marker means the generator fell behind. """ import csv import json import pathlib import statistics import sys from datetime import datetime import matplotlib matplotlib.use("Agg") import matplotlib.pyplot as plt # noqa: E402 SLO_P99_MS = 500.0 CONTROL_TOLERANCE = 0.25 # Below saturation in every configuration, so the cost is not inflated by # queueing. COST_RATES = (800, 1000) COMPONENTS = ("api", "lb", "postgres", "valkey", "loadgen") def read_jsonl(path): if not path.exists(): return [] rows = [] for line in path.read_text(encoding="utf-8").splitlines(): line = line.strip() if line: try: rows.append(json.loads(line)) except json.JSONDecodeError: pass return rows def component(name): for c in COMPONENTS: if f"-{c}-" in name: return c return None def parse_time(value): return datetime.fromisoformat(value.replace("Z", "+00:00")).timestamp() def load_runs(root): runs = [] for steps_path in sorted(root.glob("*/*/rep*/steps.json")): d = steps_path.parent report = json.loads(steps_path.read_text(encoding="utf-8")) runs.append({ "series": d.parent.parent.name, "config": d.parent.name, "rep": d.name, "start": parse_time(report["started_at"]), "steps": report["steps"], "docker": read_jsonl(d / "docker-stats.jsonl"), "metrics": read_jsonl(d / "metrics.jsonl"), "meta": json.loads((d / "meta.json").read_text(encoding="utf-8")) if (d / "meta.json").exists() else {}, }) return runs def mean_cpu(run, t0, t1): """Mean CPU per component in [t0, t1); replicas of the API are summed.""" per_time = {} for row in run["docker"]: c = component(row.get("name", "")) if c is None or not t0 <= row["t"] < t1: continue try: cpu = float(row["cpu"].rstrip("%")) except (ValueError, KeyError): continue per_time.setdefault(row["t"], {}).setdefault(c, 0.0) per_time[row["t"]][c] += cpu out = {} for c in COMPONENTS: values = [v[c] for v in per_time.values() if c in v] out[c] = round(statistics.mean(values), 1) if values else "" return out def pool_rate(run, t0, t1, counter): """Growth of a pgxpool counter during [t0, t1), all replicas, per second.""" first, last = {}, {} for row in run["metrics"]: t = run["start"] + row["t"] if not t0 <= t < t1: continue for replica in row.get("replicas") or []: pool = replica.get("pgxpool") or {} key = replica.get("instance") if key and counter in pool: first.setdefault(key, pool[counter]) last[key] = pool[counter] if not last: return "" return round(sum(last[k] - first[k] for k in last) / max(t1 - t0, 1e-9), 1) def main(root, out): runs = load_runs(root) if not runs: sys.exit(f"no runs under {root}") out.mkdir(parents=True, exist_ok=True) rows, capacities = [], [] for run in runs: best = None for step in run["steps"]: t0 = run["start"] + step["started_at_s"] t1 = t0 + step["seconds"] cpu = mean_cpu(run, t0, t1) classes = step["classes"] row = { "series": run["series"], "config": run["config"], "rep": run["rep"], "offered_rps": step["offered_rps"], "ok_rps": round(step["ok_rps"], 1), "error_share": round(step["error_share"], 4), "ok_p50_ms": step["ok_p50_ms"], "ok_p95_ms": step["ok_p95_ms"], "ok_p99_ms": step["ok_p99_ms"], "lag_p99_ms": step["lag_p99_ms"], "valid": step["valid"], "passed": step["passed"], "client_timeouts": classes.get("client_timeout", 0), "lb_errors": sum(n for c, n in classes.items() if c.endswith("_lb")), "api_errors": sum(n for c, n in classes.items() if c.startswith("http_") and not c.endswith("_lb")), **{f"cpu_{c}_pct": cpu[c] for c in COMPONENTS}, "pool_waits_per_s": pool_rate(run, t0, t1, "empty_acquire_count"), "new_conns_per_s": pool_rate(run, t0, t1, "new_conns_count"), } rows.append(row) if step["passed"] and (best is None or step["offered_rps"] > best["offered_rps"]): best = row capacities.append({ "series": run["series"], "config": run["config"], "rep": run["rep"], "capacity_rps": best["offered_rps"] if best else 0, "at": best, "invalid_below": sum(1 for s in run["steps"] if not s["valid"] and (best is None or s["offered_rps"] < best["offered_rps"])), }) with open(out / "summary.csv", "w", newline="", encoding="utf-8") as f: w = csv.DictWriter(f, fieldnames=list(rows[0])) w.writeheader() w.writerows(rows) configs = [] for c in capacities: if (c["series"], c["config"]) not in configs: configs.append((c["series"], c["config"])) with open(out / "capacity.csv", "w", newline="", encoding="utf-8") as f: w = csv.writer(f) w.writerow(["series", "config", "capacity_rps_per_repeat", "capacity_rps_median", "invalid_steps_below_capacity"]) medians = {} for key in configs: mine = [c for c in capacities if (c["series"], c["config"]) == key] values = [c["capacity_rps"] for c in mine] medians[key] = statistics.median(values) w.writerow([key[0], key[1], " ".join(f"{v:g}" for v in values), f"{medians[key]:g}", sum(c["invalid_below"] for c in mine)]) control = [] for run in runs: first = [r for r in rows if (r["series"], r["config"], r["rep"]) == (run["series"], run["config"], run["rep"])][:1] if first and first[0]["cpu_loadgen_pct"] != "": probe = run["meta"].get("host_probe_s", "") control.append([run["series"], run["config"], run["rep"], float(probe) if probe else None, round(float(first[0]["cpu_loadgen_pct"]) * 10000 / first[0]["offered_rps"])]) if control: probes = [c[3] for c in control if c[3] is not None] probe_median = statistics.median(probes) if probes else None cpu_median = statistics.median(c[4] for c in control) with open(out / "control.csv", "w", newline="", encoding="utf-8") as f: w = csv.writer(f) w.writerow(["series", "config", "rep", "host_probe_s", "probe_deviation", "loadgen_cpu_ms_per_1000_requests", "cpu_deviation", "flagged"]) flagged = set() for series, config, rep, probe, cpu in control: probe_dev = probe / probe_median - 1 if probe is not None else 0.0 cpu_dev = cpu / cpu_median - 1 noisy = max(abs(probe_dev), abs(cpu_dev)) > CONTROL_TOLERANCE if noisy: flagged.add((series, config, rep)) w.writerow([series, config, rep, probe if probe is not None else "", f"{probe_dev:+.2f}", cpu, f"{cpu_dev:+.2f}", noisy]) cost = {} for r in rows: if (r["series"], r["config"], r["rep"]) in flagged or r["offered_rps"] not in COST_RATES: continue if not r["valid"] or r["error_share"] > 0: continue per = cost.setdefault((r["series"], r["config"]), {"runs": set()}) per["runs"].add(r["rep"]) for c in ("api", "postgres", "lb"): if r[f"cpu_{c}_pct"] != "": per.setdefault(c, []).append(float(r[f"cpu_{c}_pct"]) * 10 / r["offered_rps"]) with open(out / "cost.csv", "w", newline="", encoding="utf-8") as f: w = csv.writer(f) w.writerow(["series", "config", "runs", "api_cpu_ms_per_request", "postgres_cpu_ms_per_request", "lb_cpu_ms_per_request", "rps_per_database_cpu"]) for (series, config), per in cost.items(): med = {c: statistics.median(per[c]) for c in ("api", "postgres", "lb") if per.get(c)} w.writerow([series, config, " ".join(sorted(per["runs"])), *(round(med[c], 2) if c in med else "" for c in ("api", "postgres", "lb")), round(1000 / med["postgres"]) if med.get("postgres") else ""]) if cost: keys = list(cost) fig, ax = plt.subplots(figsize=(max(6, len(keys) * 0.9), 4.5)) width = 0.27 for i, c in enumerate(("api", "postgres", "lb")): ax.bar([j + i * width for j in range(len(keys))], [statistics.median(cost[k][c]) if cost[k].get(c) else 0 for k in keys], width, label=c) ax.set_xticks([j + width for j in range(len(keys))], [f"{s_}/{c_}" for s_, c_ in keys], rotation=30) ax.set_ylabel("CPU ms per request") ax.set_title(f"Cost per request at {'/'.join(map(str, COST_RATES))} rps (clean runs)") ax.grid(axis="y", alpha=0.3) ax.legend() fig.tight_layout() fig.savefig(out / "cost-per-request.png", dpi=120) plt.close(fig) def median_by_offered(series, config, field): points, valid = {}, {} for r in rows: if r["series"] == series and r["config"] == config and r[field] != "": points.setdefault(r["offered_rps"], []).append(float(r[field])) valid[r["offered_rps"]] = valid.get(r["offered_rps"], True) and r["valid"] xs = sorted(points) return xs, [statistics.median(points[x]) for x in xs], [valid[x] for x in xs] for series in sorted({s for s, _ in configs}): names = [c for s, c in configs if s == series] for field, title, filename, log in ( ("ok_rps", "Successful requests per second", "throughput", False), ("ok_p99_ms", "p99 of successful requests, ms", "p99", True), ): fig, ax = plt.subplots(figsize=(7, 4.5)) top = 0 for name in names: xs, ys, valid = median_by_offered(series, name, field) line, = ax.plot(xs, ys, label=name) ax.scatter([x for x, v in zip(xs, valid) if v], [y for y, v in zip(ys, valid) if v], color=line.get_color()) ax.scatter([x for x, v in zip(xs, valid) if not v], [y for y, v in zip(ys, valid) if not v], facecolors="none", edgecolors=line.get_color()) top = max([top, *xs]) if field == "ok_rps" and top: ax.plot([0, top], [0, top], linestyle=":", color="grey", label="offered") if field == "ok_p99_ms": ax.axhline(SLO_P99_MS, linestyle="--", color="grey", label="500 ms") if log: ax.set_yscale("log") ax.set_xlabel("Offered requests per second") ax.set_ylabel(title) ax.set_title(f"Series {series}: median of repeats (hollow: generator fell behind)") ax.grid(alpha=0.3) ax.legend() fig.tight_layout() fig.savefig(out / f"{series}-{filename}.png", dpi=120) plt.close(fig) labels = [f"{s}/{c}" for s, c in configs] fig, ax = plt.subplots(figsize=(max(6, len(labels) * 0.9), 4.5)) meds = [medians[k] for k in configs] lows = [medians[k] - min(c["capacity_rps"] for c in capacities if (c["series"], c["config"]) == k) for k in configs] highs = [max(c["capacity_rps"] for c in capacities if (c["series"], c["config"]) == k) - medians[k] for k in configs] ax.bar(labels, meds, yerr=[lows, highs], capsize=4, color="#4c78a8") ax.set_ylabel("Capacity, requests per second") ax.set_title("Capacity per configuration (median, range of repeats)") ax.tick_params(axis="x", rotation=30) ax.grid(axis="y", alpha=0.3) fig.tight_layout() fig.savefig(out / "capacity.png", dpi=120) plt.close(fig) fig, ax = plt.subplots(figsize=(max(6, len(labels) * 1.1), 4.5)) width = 0.8 / len(COMPONENTS) for i, comp in enumerate(COMPONENTS): values = [] for key in configs: at = [c["at"] for c in capacities if (c["series"], c["config"]) == key and c["at"]] cpu = [float(a[f"cpu_{comp}_pct"]) for a in at if a[f"cpu_{comp}_pct"] != ""] values.append(statistics.median(cpu) if cpu else 0) ax.bar([j + i * width for j in range(len(configs))], values, width, label=comp) ax.set_xticks([j + 0.4 - width / 2 for j in range(len(configs))], labels, rotation=30) ax.set_ylabel("Mean CPU at capacity, % of one core") ax.set_title("Which component is busy at capacity") ax.grid(axis="y", alpha=0.3) ax.legend() fig.tight_layout() fig.savefig(out / "cpu-at-capacity.png", dpi=120) plt.close(fig) fig, ax = plt.subplots(figsize=(7, 4.5)) for series, name in configs: xs, ys, _ = median_by_offered(series, name, "pool_waits_per_s") if xs: ax.plot(xs, ys, marker="o", label=f"{series}/{name}") ax.set_xlabel("Offered requests per second") ax.set_ylabel("Waits for a pooled connection per second") ax.set_title("Connection pool waits, all replicas") ax.grid(alpha=0.3) ax.legend() fig.tight_layout() fig.savefig(out / "pool-waits.png", dpi=120) plt.close(fig) print(f"{len(runs)} runs, {len(rows)} steps -> {out}") if __name__ == "__main__": if len(sys.argv) != 3: sys.exit(__doc__) main(pathlib.Path(sys.argv[1]), pathlib.Path(sys.argv[2]))