#!/usr/bin/env python3 """Tables and charts of track 1 from the raw results of run-series.sh. python scripts/bench/plot.py The raw root holds //rep/ directories. Written to the output directory: summary.csv one row per step of every run; capacity.csv the capacity of every run and the median per configuration; control.csv the noise checks of every run, the host speed probe and the loadgen CPU per request at the first step, each flagged when more than 25 % off its median; cost.csv CPU milliseconds per request of each component at the COST_RATES steps (valid, error-free, unflagged runs only), and the rate one database CPU would serve at that cost; -throughput.png, -p99.png, capacity.png, cpu-at-capacity.png, pool-waits.png, cost-per-request.png. Capacity is the highest step that passed the criterion loadgen applies (valid, under 1 % errors, p99 of successful requests within 500 ms). Charts show the median of the repeats; a hollow marker means the generator fell behind. """ import csv import gzip import json import pathlib import statistics import sys from datetime import datetime import matplotlib matplotlib.use("Agg") import matplotlib.pyplot as plt # noqa: E402 SLO_P99_MS = 500.0 CONTROL_TOLERANCE = 0.25 # Below saturation in every configuration, so the cost is not inflated by # queueing. COST_RATES = (800, 1000) COMPONENTS = ("api", "lb", "postgres", "valkey", "loadgen") def read_jsonl(path): packed = path.with_name(path.name + ".gz") if packed.exists(): text = gzip.decompress(packed.read_bytes()).decode("utf-8") elif path.exists(): text = path.read_text(encoding="utf-8") else: return [] rows = [] for line in text.splitlines(): line = line.strip() if line: try: rows.append(json.loads(line)) except json.JSONDecodeError: pass return rows def component(name): for c in COMPONENTS: if f"-{c}-" in name: return c return None def parse_time(value): return datetime.fromisoformat(value.replace("Z", "+00:00")).timestamp() def load_runs(root): runs = [] for steps_path in sorted(root.glob("*/*/rep*/steps.json")): d = steps_path.parent report = json.loads(steps_path.read_text(encoding="utf-8")) runs.append({ "series": d.parent.parent.name, "config": d.parent.name, "rep": d.name, "start": parse_time(report["started_at"]), "steps": report["steps"], "docker": read_jsonl(d / "docker-stats.jsonl"), "metrics": read_jsonl(d / "metrics.jsonl"), "meta": json.loads((d / "meta.json").read_text(encoding="utf-8")) if (d / "meta.json").exists() else {}, }) return runs def mean_cpu(run, t0, t1): """Mean CPU per component in [t0, t1); replicas of the API are summed.""" per_time = {} for row in run["docker"]: c = component(row.get("name", "")) if c is None or not t0 <= row["t"] < t1: continue try: cpu = float(row["cpu"].rstrip("%")) except (ValueError, KeyError): continue per_time.setdefault(row["t"], {}).setdefault(c, 0.0) per_time[row["t"]][c] += cpu out = {} for c in COMPONENTS: values = [v[c] for v in per_time.values() if c in v] out[c] = round(statistics.mean(values), 1) if values else "" return out def pool_rate(run, t0, t1, counter): """Growth of a pgxpool counter during [t0, t1), all replicas, per second.""" first, last = {}, {} for row in run["metrics"]: t = run["start"] + row["t"] if not t0 <= t < t1: continue for replica in row.get("replicas") or []: pool = replica.get("pgxpool") or {} key = replica.get("instance") if key and counter in pool: first.setdefault(key, pool[counter]) last[key] = pool[counter] if not last: return "" return round(sum(last[k] - first[k] for k in last) / max(t1 - t0, 1e-9), 1) def main(root, out): runs = load_runs(root) if not runs: sys.exit(f"no runs under {root}") out.mkdir(parents=True, exist_ok=True) rows, capacities = [], [] for run in runs: best = None for step in run["steps"]: t0 = run["start"] + step["started_at_s"] t1 = t0 + step["seconds"] cpu = mean_cpu(run, t0, t1) classes = step["classes"] row = { "series": run["series"], "config": run["config"], "rep": run["rep"], "offered_rps": step["offered_rps"], "ok_rps": round(step["ok_rps"], 1), "error_share": round(step["error_share"], 4), "ok_p50_ms": step["ok_p50_ms"], "ok_p95_ms": step["ok_p95_ms"], "ok_p99_ms": step["ok_p99_ms"], "lag_p99_ms": step["lag_p99_ms"], "valid": step["valid"], "passed": step["passed"], "client_timeouts": classes.get("client_timeout", 0), "lb_errors": sum(n for c, n in classes.items() if c.endswith("_lb")), "api_errors": sum(n for c, n in classes.items() if c.startswith("http_") and not c.endswith("_lb")), **{f"cpu_{c}_pct": cpu[c] for c in COMPONENTS}, "pool_waits_per_s": pool_rate(run, t0, t1, "empty_acquire_count"), "new_conns_per_s": pool_rate(run, t0, t1, "new_conns_count"), } rows.append(row) if step["passed"] and (best is None or step["offered_rps"] > best["offered_rps"]): best = row capacities.append({ "series": run["series"], "config": run["config"], "rep": run["rep"], "capacity_rps": best["offered_rps"] if best else 0, "at": best, "invalid_below": sum(1 for s in run["steps"] if not s["valid"] and (best is None or s["offered_rps"] < best["offered_rps"])), }) with open(out / "summary.csv", "w", newline="", encoding="utf-8") as f: w = csv.DictWriter(f, fieldnames=list(rows[0])) w.writeheader() w.writerows(rows) configs = [] for c in capacities: if (c["series"], c["config"]) not in configs: configs.append((c["series"], c["config"])) with open(out / "capacity.csv", "w", newline="", encoding="utf-8") as f: w = csv.writer(f) w.writerow(["series", "config", "capacity_rps_per_repeat", "capacity_rps_median", "invalid_steps_below_capacity"]) medians = {} for key in configs: mine = [c for c in capacities if (c["series"], c["config"]) == key] values = [c["capacity_rps"] for c in mine] medians[key] = statistics.median(values) w.writerow([key[0], key[1], " ".join(f"{v:g}" for v in values), f"{medians[key]:g}", sum(c["invalid_below"] for c in mine)]) control = [] for run in runs: first = [r for r in rows if (r["series"], r["config"], r["rep"]) == (run["series"], run["config"], run["rep"])][:1] if first and first[0]["cpu_loadgen_pct"] != "": probe = run["meta"].get("host_probe_s", "") control.append([run["series"], run["config"], run["rep"], float(probe) if probe else None, round(float(first[0]["cpu_loadgen_pct"]) * 10000 / first[0]["offered_rps"])]) if control: probes = [c[3] for c in control if c[3] is not None] probe_median = statistics.median(probes) if probes else None cpu_median = statistics.median(c[4] for c in control) with open(out / "control.csv", "w", newline="", encoding="utf-8") as f: w = csv.writer(f) w.writerow(["series", "config", "rep", "host_probe_s", "probe_deviation", "loadgen_cpu_ms_per_1000_requests", "cpu_deviation", "flagged"]) flagged = set() for series, config, rep, probe, cpu in control: probe_dev = probe / probe_median - 1 if probe is not None else 0.0 cpu_dev = cpu / cpu_median - 1 noisy = max(abs(probe_dev), abs(cpu_dev)) > CONTROL_TOLERANCE if noisy: flagged.add((series, config, rep)) w.writerow([series, config, rep, probe if probe is not None else "", f"{probe_dev:+.2f}", cpu, f"{cpu_dev:+.2f}", noisy]) cost = {} for r in rows: if (r["series"], r["config"], r["rep"]) in flagged or r["offered_rps"] not in COST_RATES: continue if not r["valid"] or r["error_share"] > 0: continue per = cost.setdefault((r["series"], r["config"]), {"runs": set()}) per["runs"].add(r["rep"]) for c in ("api", "postgres", "lb"): if r[f"cpu_{c}_pct"] != "": per.setdefault(c, []).append(float(r[f"cpu_{c}_pct"]) * 10 / r["offered_rps"]) with open(out / "cost.csv", "w", newline="", encoding="utf-8") as f: w = csv.writer(f) w.writerow(["series", "config", "runs", "api_cpu_ms_per_request", "postgres_cpu_ms_per_request", "lb_cpu_ms_per_request", "rps_per_database_cpu"]) for (series, config), per in cost.items(): med = {c: statistics.median(per[c]) for c in ("api", "postgres", "lb") if per.get(c)} w.writerow([series, config, " ".join(sorted(per["runs"])), *(round(med[c], 2) if c in med else "" for c in ("api", "postgres", "lb")), round(1000 / med["postgres"]) if med.get("postgres") else ""]) if cost: keys = list(cost) fig, ax = plt.subplots(figsize=(max(6, len(keys) * 0.9), 4.5)) width = 0.27 for i, c in enumerate(("api", "postgres", "lb")): ax.bar([j + i * width for j in range(len(keys))], [statistics.median(cost[k][c]) if cost[k].get(c) else 0 for k in keys], width, label=c) ax.set_xticks([j + width for j in range(len(keys))], [f"{s_}/{c_}" for s_, c_ in keys], rotation=30) ax.set_ylabel("CPU ms per request") ax.set_title(f"Cost per request at {'/'.join(map(str, COST_RATES))} rps (clean runs)") ax.grid(axis="y", alpha=0.3) ax.legend() fig.tight_layout() fig.savefig(out / "cost-per-request.png", dpi=120) plt.close(fig) def median_by_offered(series, config, field): points, valid = {}, {} for r in rows: if r["series"] == series and r["config"] == config and r[field] != "": points.setdefault(r["offered_rps"], []).append(float(r[field])) valid[r["offered_rps"]] = valid.get(r["offered_rps"], True) and r["valid"] xs = sorted(points) return xs, [statistics.median(points[x]) for x in xs], [valid[x] for x in xs] for series in sorted({s for s, _ in configs}): names = [c for s, c in configs if s == series] for field, title, filename, log in ( ("ok_rps", "Successful requests per second", "throughput", False), ("ok_p99_ms", "p99 of successful requests, ms", "p99", True), ): fig, ax = plt.subplots(figsize=(7, 4.5)) top = 0 for name in names: xs, ys, valid = median_by_offered(series, name, field) line, = ax.plot(xs, ys, label=name) ax.scatter([x for x, v in zip(xs, valid) if v], [y for y, v in zip(ys, valid) if v], color=line.get_color()) ax.scatter([x for x, v in zip(xs, valid) if not v], [y for y, v in zip(ys, valid) if not v], facecolors="none", edgecolors=line.get_color()) top = max([top, *xs]) if field == "ok_rps" and top: ax.plot([0, top], [0, top], linestyle=":", color="grey", label="offered") if field == "ok_p99_ms": ax.axhline(SLO_P99_MS, linestyle="--", color="grey", label="500 ms") if log: ax.set_yscale("log") ax.set_xlabel("Offered requests per second") ax.set_ylabel(title) ax.set_title(f"Series {series}: median of repeats (hollow: generator fell behind)") ax.grid(alpha=0.3) ax.legend() fig.tight_layout() fig.savefig(out / f"{series}-{filename}.png", dpi=120) plt.close(fig) labels = [f"{s}/{c}" for s, c in configs] fig, ax = plt.subplots(figsize=(max(6, len(labels) * 0.9), 4.5)) meds = [medians[k] for k in configs] lows = [medians[k] - min(c["capacity_rps"] for c in capacities if (c["series"], c["config"]) == k) for k in configs] highs = [max(c["capacity_rps"] for c in capacities if (c["series"], c["config"]) == k) - medians[k] for k in configs] ax.bar(labels, meds, yerr=[lows, highs], capsize=4, color="#4c78a8") ax.set_ylabel("Capacity, requests per second") ax.set_title("Capacity per configuration (median, range of repeats)") ax.tick_params(axis="x", rotation=30) ax.grid(axis="y", alpha=0.3) fig.tight_layout() fig.savefig(out / "capacity.png", dpi=120) plt.close(fig) fig, ax = plt.subplots(figsize=(max(6, len(labels) * 1.1), 4.5)) width = 0.8 / len(COMPONENTS) for i, comp in enumerate(COMPONENTS): values = [] for key in configs: at = [c["at"] for c in capacities if (c["series"], c["config"]) == key and c["at"]] cpu = [float(a[f"cpu_{comp}_pct"]) for a in at if a[f"cpu_{comp}_pct"] != ""] values.append(statistics.median(cpu) if cpu else 0) ax.bar([j + i * width for j in range(len(configs))], values, width, label=comp) ax.set_xticks([j + 0.4 - width / 2 for j in range(len(configs))], labels, rotation=30) ax.set_ylabel("Mean CPU at capacity, % of one core") ax.set_title("Which component is busy at capacity") ax.grid(axis="y", alpha=0.3) ax.legend() fig.tight_layout() fig.savefig(out / "cpu-at-capacity.png", dpi=120) plt.close(fig) fig, ax = plt.subplots(figsize=(7, 4.5)) for series, name in configs: xs, ys, _ = median_by_offered(series, name, "pool_waits_per_s") if xs: ax.plot(xs, ys, marker="o", label=f"{series}/{name}") ax.set_xlabel("Offered requests per second") ax.set_ylabel("Waits for a pooled connection per second") ax.set_title("Connection pool waits, all replicas") ax.grid(alpha=0.3) ax.legend() fig.tight_layout() fig.savefig(out / "pool-waits.png", dpi=120) plt.close(fig) print(f"{len(runs)} runs, {len(rows)} steps -> {out}") if __name__ == "__main__": if len(sys.argv) != 3: sys.exit(__doc__) main(pathlib.Path(sys.argv[1]), pathlib.Path(sys.argv[2]))