Files
mtusi2026lab3/scripts/bench/plot.py
T

254 lines
9.5 KiB
Python
Executable File

#!/usr/bin/env python3
"""Tables and charts of track 1 from the raw results of run-series.sh.
python scripts/bench/plot.py <raw root> <output dir>
The raw root holds <series>/<config>/rep<k>/ directories. Written to the
output directory:
summary.csv one row per step of every run: offered and successful rate,
errors, latency percentiles, generator lag, mean CPU of each
component and pool waits during the step;
capacity.csv the capacity of every run and the median per configuration;
<series>-throughput.png, <series>-p99.png, capacity.png,
cpu-at-capacity.png, pool-waits.png.
Capacity is the highest valid step (generator on schedule) with less than 1 %
errors and a p99 of successful requests within 500 ms.
"""
import csv
import json
import pathlib
import statistics
import sys
from datetime import datetime
import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt # noqa: E402
SLO_P99_MS = 500.0
MAX_ERROR_SHARE = 0.01
COMPONENTS = ("api", "lb", "postgres", "valkey", "loadgen")
def read_jsonl(path):
if not path.exists():
return []
rows = []
for line in path.read_text(encoding="utf-8").splitlines():
line = line.strip()
if line:
try:
rows.append(json.loads(line))
except json.JSONDecodeError:
pass
return rows
def component(name):
for c in COMPONENTS:
if f"-{c}-" in name:
return c
return None
def parse_time(value):
return datetime.fromisoformat(value.replace("Z", "+00:00")).timestamp()
def load_runs(root):
runs = []
for steps_path in sorted(root.glob("*/*/rep*/steps.json")):
d = steps_path.parent
report = json.loads(steps_path.read_text(encoding="utf-8"))
runs.append({
"series": d.parent.parent.name,
"config": d.parent.name,
"rep": d.name,
"start": parse_time(report["started_at"]),
"steps": report["steps"],
"docker": read_jsonl(d / "docker-stats.jsonl"),
"metrics": read_jsonl(d / "metrics.jsonl"),
})
return runs
def mean_cpu(run, t0, t1):
"""Mean CPU per component in [t0, t1); replicas of the API are summed."""
per_time = {}
for row in run["docker"]:
c = component(row.get("name", ""))
if c is None or not t0 <= row["t"] < t1:
continue
try:
cpu = float(row["cpu"].rstrip("%"))
except (ValueError, KeyError):
continue
per_time.setdefault(row["t"], {}).setdefault(c, 0.0)
per_time[row["t"]][c] += cpu
out = {}
for c in COMPONENTS:
values = [v[c] for v in per_time.values() if c in v]
out[c] = round(statistics.mean(values), 1) if values else ""
return out
def pool_waits(run, t0, t1):
"""Requests that waited for a pooled connection during [t0, t1), all
replicas together, per second."""
first, last = {}, {}
for row in run["metrics"]:
t = run["start"] + row["t"]
if not t0 <= t < t1:
continue
for replica in row.get("replicas") or []:
pool = replica.get("pgxpool") or {}
key = replica.get("instance")
if key and "empty_acquire_count" in pool:
first.setdefault(key, pool["empty_acquire_count"])
last[key] = pool["empty_acquire_count"]
if not last:
return ""
return round(sum(last[k] - first[k] for k in last) / max(t1 - t0, 1e-9), 1)
def is_capacity(step):
return step["valid"] and step["error_share"] < MAX_ERROR_SHARE and step["ok_p99_ms"] <= SLO_P99_MS
def main(root, out):
runs = load_runs(root)
if not runs:
sys.exit(f"no runs under {root}")
out.mkdir(parents=True, exist_ok=True)
rows, capacities = [], []
for run in runs:
best = None
for step in run["steps"]:
t0 = run["start"] + step["started_at_s"]
t1 = t0 + step["seconds"]
cpu = mean_cpu(run, t0, t1)
row = {
"series": run["series"], "config": run["config"], "rep": run["rep"],
"offered_rps": step["offered_rps"], "ok_rps": round(step["ok_rps"], 1),
"error_share": round(step["error_share"], 4), "ok_p50_ms": step["ok_p50_ms"],
"ok_p95_ms": step["ok_p95_ms"], "ok_p99_ms": step["ok_p99_ms"],
"lag_p99_ms": step["lag_p99_ms"], "valid": step["valid"],
**{f"cpu_{c}_pct": cpu[c] for c in COMPONENTS},
"pool_waits_per_s": pool_waits(run, t0, t1),
}
rows.append(row)
if is_capacity(step):
best = row
capacities.append({
"series": run["series"], "config": run["config"], "rep": run["rep"],
"capacity_rps": best["offered_rps"] if best else 0, "at": best,
})
with open(out / "summary.csv", "w", newline="", encoding="utf-8") as f:
w = csv.DictWriter(f, fieldnames=list(rows[0]))
w.writeheader()
w.writerows(rows)
configs = []
for c in capacities:
if (c["series"], c["config"]) not in configs:
configs.append((c["series"], c["config"]))
with open(out / "capacity.csv", "w", newline="", encoding="utf-8") as f:
w = csv.writer(f)
w.writerow(["series", "config", "capacity_rps_per_repeat", "capacity_rps_median"])
medians = {}
for key in configs:
values = [c["capacity_rps"] for c in capacities if (c["series"], c["config"]) == key]
medians[key] = statistics.median(values)
w.writerow([key[0], key[1], " ".join(f"{v:g}" for v in values), f"{medians[key]:g}"])
def median_by_offered(series, config, field):
points = {}
for r in rows:
if r["series"] == series and r["config"] == config and r["valid"] and r[field] != "":
points.setdefault(r["offered_rps"], []).append(float(r[field]))
xs = sorted(points)
return xs, [statistics.median(points[x]) for x in xs]
for series in sorted({s for s, _ in configs}):
names = [c for s, c in configs if s == series]
for field, title, filename, log in (
("ok_rps", "Successful requests per second", "throughput", False),
("ok_p99_ms", "p99 of successful requests, ms", "p99", True),
):
fig, ax = plt.subplots(figsize=(7, 4.5))
top = 0
for name in names:
xs, ys = median_by_offered(series, name, field)
ax.plot(xs, ys, marker="o", label=name)
top = max([top, *xs])
if field == "ok_rps" and top:
ax.plot([0, top], [0, top], linestyle=":", color="grey", label="offered")
if field == "ok_p99_ms":
ax.axhline(SLO_P99_MS, linestyle="--", color="grey", label="500 ms")
if log:
ax.set_yscale("log")
ax.set_xlabel("Offered requests per second")
ax.set_ylabel(title)
ax.set_title(f"Series {series}: median of repeats, valid steps only")
ax.grid(alpha=0.3)
ax.legend()
fig.tight_layout()
fig.savefig(out / f"{series}-{filename}.png", dpi=120)
plt.close(fig)
labels = [f"{s}/{c}" for s, c in configs]
fig, ax = plt.subplots(figsize=(max(6, len(labels) * 0.9), 4.5))
meds = [medians[k] for k in configs]
lows = [medians[k] - min(c["capacity_rps"] for c in capacities if (c["series"], c["config"]) == k) for k in configs]
highs = [max(c["capacity_rps"] for c in capacities if (c["series"], c["config"]) == k) - medians[k] for k in configs]
ax.bar(labels, meds, yerr=[lows, highs], capsize=4, color="#4c78a8")
ax.set_ylabel("Capacity, requests per second")
ax.set_title("Capacity per configuration (median, range of repeats)")
ax.tick_params(axis="x", rotation=30)
ax.grid(axis="y", alpha=0.3)
fig.tight_layout()
fig.savefig(out / "capacity.png", dpi=120)
plt.close(fig)
fig, ax = plt.subplots(figsize=(max(6, len(labels) * 1.1), 4.5))
width = 0.8 / len(COMPONENTS)
for i, comp in enumerate(COMPONENTS):
values = []
for key in configs:
at = [c["at"] for c in capacities if (c["series"], c["config"]) == key and c["at"]]
cpu = [float(a[f"cpu_{comp}_pct"]) for a in at if a[f"cpu_{comp}_pct"] != ""]
values.append(statistics.median(cpu) if cpu else 0)
ax.bar([j + i * width for j in range(len(configs))], values, width, label=comp)
ax.set_xticks([j + 0.4 - width / 2 for j in range(len(configs))], labels, rotation=30)
ax.set_ylabel("Mean CPU at capacity, % of one core")
ax.set_title("Which component is busy at capacity")
ax.grid(axis="y", alpha=0.3)
ax.legend()
fig.tight_layout()
fig.savefig(out / "cpu-at-capacity.png", dpi=120)
plt.close(fig)
fig, ax = plt.subplots(figsize=(7, 4.5))
for series, name in configs:
xs, ys = median_by_offered(series, name, "pool_waits_per_s")
if xs:
ax.plot(xs, ys, marker="o", label=f"{series}/{name}")
ax.set_xlabel("Offered requests per second")
ax.set_ylabel("Waits for a pooled connection per second")
ax.set_title("Connection pool waits, all replicas")
ax.grid(alpha=0.3)
ax.legend()
fig.tight_layout()
fig.savefig(out / "pool-waits.png", dpi=120)
plt.close(fig)
print(f"{len(runs)} runs, {len(rows)} steps -> {out}")
if __name__ == "__main__":
if len(sys.argv) != 3:
sys.exit(__doc__)
main(pathlib.Path(sys.argv[1]), pathlib.Path(sys.argv[2]))