Files
mtusi2026lab3/scripts/bench/plot.py
T

337 lines
15 KiB
Python
Executable File

#!/usr/bin/env python3
"""Tables and charts of track 1 from the raw results of run-series.sh.
python scripts/bench/plot.py <raw root> <output dir>
The raw root holds <series>/<config>/rep<k>/ directories. Written to the
output directory:
summary.csv one row per step of every run;
capacity.csv the capacity of every run and the median per configuration;
control.csv the noise checks of every run, the host speed probe and the
loadgen CPU per request at the first step, each flagged when
more than 25 % off its median;
cost.csv CPU milliseconds per request of each component at the
COST_RATES steps (valid, error-free, unflagged runs only), and
the rate one database CPU would serve at that cost;
<series>-throughput.png, <series>-p99.png, capacity.png,
cpu-at-capacity.png, pool-waits.png, cost-per-request.png.
Capacity is the highest step that passed the criterion loadgen applies (valid,
under 1 % errors, p99 of successful requests within 500 ms). Charts show the
median of the repeats; a hollow marker means the generator fell behind.
"""
import csv
import gzip
import json
import pathlib
import statistics
import sys
from datetime import datetime
import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt # noqa: E402
SLO_P99_MS = 500.0
CONTROL_TOLERANCE = 0.25
# Below saturation in every configuration, so the cost is not inflated by
# queueing.
COST_RATES = (800, 1000)
COMPONENTS = ("api", "lb", "postgres", "valkey", "loadgen")
def read_jsonl(path):
packed = path.with_name(path.name + ".gz")
if packed.exists():
text = gzip.decompress(packed.read_bytes()).decode("utf-8")
elif path.exists():
text = path.read_text(encoding="utf-8")
else:
return []
rows = []
for line in text.splitlines():
line = line.strip()
if line:
try:
rows.append(json.loads(line))
except json.JSONDecodeError:
pass
return rows
def component(name):
for c in COMPONENTS:
if f"-{c}-" in name:
return c
return None
def parse_time(value):
return datetime.fromisoformat(value.replace("Z", "+00:00")).timestamp()
def load_runs(root):
runs = []
for steps_path in sorted(root.glob("*/*/rep*/steps.json")):
d = steps_path.parent
report = json.loads(steps_path.read_text(encoding="utf-8"))
runs.append({
"series": d.parent.parent.name,
"config": d.parent.name,
"rep": d.name,
"start": parse_time(report["started_at"]),
"steps": report["steps"],
"docker": read_jsonl(d / "docker-stats.jsonl"),
"metrics": read_jsonl(d / "metrics.jsonl"),
"meta": json.loads((d / "meta.json").read_text(encoding="utf-8")) if (d / "meta.json").exists() else {},
})
return runs
def mean_cpu(run, t0, t1):
"""Mean CPU per component in [t0, t1); replicas of the API are summed."""
per_time = {}
for row in run["docker"]:
c = component(row.get("name", ""))
if c is None or not t0 <= row["t"] < t1:
continue
try:
cpu = float(row["cpu"].rstrip("%"))
except (ValueError, KeyError):
continue
per_time.setdefault(row["t"], {}).setdefault(c, 0.0)
per_time[row["t"]][c] += cpu
out = {}
for c in COMPONENTS:
values = [v[c] for v in per_time.values() if c in v]
out[c] = round(statistics.mean(values), 1) if values else ""
return out
def pool_rate(run, t0, t1, counter):
"""Growth of a pgxpool counter during [t0, t1), all replicas, per second."""
first, last = {}, {}
for row in run["metrics"]:
t = run["start"] + row["t"]
if not t0 <= t < t1:
continue
for replica in row.get("replicas") or []:
pool = replica.get("pgxpool") or {}
key = replica.get("instance")
if key and counter in pool:
first.setdefault(key, pool[counter])
last[key] = pool[counter]
if not last:
return ""
return round(sum(last[k] - first[k] for k in last) / max(t1 - t0, 1e-9), 1)
def main(root, out):
runs = load_runs(root)
if not runs:
sys.exit(f"no runs under {root}")
out.mkdir(parents=True, exist_ok=True)
rows, capacities = [], []
for run in runs:
best = None
for step in run["steps"]:
t0 = run["start"] + step["started_at_s"]
t1 = t0 + step["seconds"]
cpu = mean_cpu(run, t0, t1)
classes = step["classes"]
row = {
"series": run["series"], "config": run["config"], "rep": run["rep"],
"offered_rps": step["offered_rps"], "ok_rps": round(step["ok_rps"], 1),
"error_share": round(step["error_share"], 4), "ok_p50_ms": step["ok_p50_ms"],
"ok_p95_ms": step["ok_p95_ms"], "ok_p99_ms": step["ok_p99_ms"],
"lag_p99_ms": step["lag_p99_ms"], "valid": step["valid"], "passed": step["passed"],
"client_timeouts": classes.get("client_timeout", 0),
"lb_errors": sum(n for c, n in classes.items() if c.endswith("_lb")),
"api_errors": sum(n for c, n in classes.items() if c.startswith("http_") and not c.endswith("_lb")),
**{f"cpu_{c}_pct": cpu[c] for c in COMPONENTS},
"pool_waits_per_s": pool_rate(run, t0, t1, "empty_acquire_count"),
"new_conns_per_s": pool_rate(run, t0, t1, "new_conns_count"),
}
rows.append(row)
if step["passed"] and (best is None or step["offered_rps"] > best["offered_rps"]):
best = row
capacities.append({
"series": run["series"], "config": run["config"], "rep": run["rep"],
"capacity_rps": best["offered_rps"] if best else 0, "at": best,
"invalid_below": sum(1 for s in run["steps"] if not s["valid"] and (best is None or s["offered_rps"] < best["offered_rps"])),
})
with open(out / "summary.csv", "w", newline="", encoding="utf-8") as f:
w = csv.DictWriter(f, fieldnames=list(rows[0]))
w.writeheader()
w.writerows(rows)
configs = []
for c in capacities:
if (c["series"], c["config"]) not in configs:
configs.append((c["series"], c["config"]))
with open(out / "capacity.csv", "w", newline="", encoding="utf-8") as f:
w = csv.writer(f)
w.writerow(["series", "config", "capacity_rps_per_repeat", "capacity_rps_median", "invalid_steps_below_capacity"])
medians = {}
for key in configs:
mine = [c for c in capacities if (c["series"], c["config"]) == key]
values = [c["capacity_rps"] for c in mine]
medians[key] = statistics.median(values)
w.writerow([key[0], key[1], " ".join(f"{v:g}" for v in values), f"{medians[key]:g}",
sum(c["invalid_below"] for c in mine)])
control = []
for run in runs:
first = [r for r in rows if (r["series"], r["config"], r["rep"]) == (run["series"], run["config"], run["rep"])][:1]
if first and first[0]["cpu_loadgen_pct"] != "":
probe = run["meta"].get("host_probe_s", "")
control.append([run["series"], run["config"], run["rep"], float(probe) if probe else None,
round(float(first[0]["cpu_loadgen_pct"]) * 10000 / first[0]["offered_rps"])])
if control:
probes = [c[3] for c in control if c[3] is not None]
probe_median = statistics.median(probes) if probes else None
cpu_median = statistics.median(c[4] for c in control)
with open(out / "control.csv", "w", newline="", encoding="utf-8") as f:
w = csv.writer(f)
w.writerow(["series", "config", "rep", "host_probe_s", "probe_deviation",
"loadgen_cpu_ms_per_1000_requests", "cpu_deviation", "flagged"])
flagged = set()
for series, config, rep, probe, cpu in control:
probe_dev = probe / probe_median - 1 if probe is not None else 0.0
cpu_dev = cpu / cpu_median - 1
noisy = max(abs(probe_dev), abs(cpu_dev)) > CONTROL_TOLERANCE
if noisy:
flagged.add((series, config, rep))
w.writerow([series, config, rep, probe if probe is not None else "", f"{probe_dev:+.2f}", cpu,
f"{cpu_dev:+.2f}", noisy])
cost = {}
for r in rows:
if (r["series"], r["config"], r["rep"]) in flagged or r["offered_rps"] not in COST_RATES:
continue
if not r["valid"] or r["error_share"] > 0:
continue
per = cost.setdefault((r["series"], r["config"]), {"runs": set()})
per["runs"].add(r["rep"])
for c in ("api", "postgres", "lb"):
if r[f"cpu_{c}_pct"] != "":
per.setdefault(c, []).append(float(r[f"cpu_{c}_pct"]) * 10 / r["offered_rps"])
with open(out / "cost.csv", "w", newline="", encoding="utf-8") as f:
w = csv.writer(f)
w.writerow(["series", "config", "runs", "api_cpu_ms_per_request", "postgres_cpu_ms_per_request",
"lb_cpu_ms_per_request", "rps_per_database_cpu"])
for (series, config), per in cost.items():
med = {c: statistics.median(per[c]) for c in ("api", "postgres", "lb") if per.get(c)}
w.writerow([series, config, " ".join(sorted(per["runs"])),
*(round(med[c], 2) if c in med else "" for c in ("api", "postgres", "lb")),
round(1000 / med["postgres"]) if med.get("postgres") else ""])
if cost:
keys = list(cost)
fig, ax = plt.subplots(figsize=(max(6, len(keys) * 0.9), 4.5))
width = 0.27
for i, c in enumerate(("api", "postgres", "lb")):
ax.bar([j + i * width for j in range(len(keys))],
[statistics.median(cost[k][c]) if cost[k].get(c) else 0 for k in keys], width, label=c)
ax.set_xticks([j + width for j in range(len(keys))], [f"{s_}/{c_}" for s_, c_ in keys], rotation=30)
ax.set_ylabel("CPU ms per request")
ax.set_title(f"Cost per request at {'/'.join(map(str, COST_RATES))} rps (clean runs)")
ax.grid(axis="y", alpha=0.3)
ax.legend()
fig.tight_layout()
fig.savefig(out / "cost-per-request.png", dpi=120)
plt.close(fig)
def median_by_offered(series, config, field):
points, valid = {}, {}
for r in rows:
if r["series"] == series and r["config"] == config and r[field] != "":
points.setdefault(r["offered_rps"], []).append(float(r[field]))
valid[r["offered_rps"]] = valid.get(r["offered_rps"], True) and r["valid"]
xs = sorted(points)
return xs, [statistics.median(points[x]) for x in xs], [valid[x] for x in xs]
for series in sorted({s for s, _ in configs}):
names = [c for s, c in configs if s == series]
for field, title, filename, log in (
("ok_rps", "Successful requests per second", "throughput", False),
("ok_p99_ms", "p99 of successful requests, ms", "p99", True),
):
fig, ax = plt.subplots(figsize=(7, 4.5))
top = 0
for name in names:
xs, ys, valid = median_by_offered(series, name, field)
line, = ax.plot(xs, ys, label=name)
ax.scatter([x for x, v in zip(xs, valid) if v], [y for y, v in zip(ys, valid) if v], color=line.get_color())
ax.scatter([x for x, v in zip(xs, valid) if not v], [y for y, v in zip(ys, valid) if not v],
facecolors="none", edgecolors=line.get_color())
top = max([top, *xs])
if field == "ok_rps" and top:
ax.plot([0, top], [0, top], linestyle=":", color="grey", label="offered")
if field == "ok_p99_ms":
ax.axhline(SLO_P99_MS, linestyle="--", color="grey", label="500 ms")
if log:
ax.set_yscale("log")
ax.set_xlabel("Offered requests per second")
ax.set_ylabel(title)
ax.set_title(f"Series {series}: median of repeats (hollow: generator fell behind)")
ax.grid(alpha=0.3)
ax.legend()
fig.tight_layout()
fig.savefig(out / f"{series}-{filename}.png", dpi=120)
plt.close(fig)
labels = [f"{s}/{c}" for s, c in configs]
fig, ax = plt.subplots(figsize=(max(6, len(labels) * 0.9), 4.5))
meds = [medians[k] for k in configs]
lows = [medians[k] - min(c["capacity_rps"] for c in capacities if (c["series"], c["config"]) == k) for k in configs]
highs = [max(c["capacity_rps"] for c in capacities if (c["series"], c["config"]) == k) - medians[k] for k in configs]
ax.bar(labels, meds, yerr=[lows, highs], capsize=4, color="#4c78a8")
ax.set_ylabel("Capacity, requests per second")
ax.set_title("Capacity per configuration (median, range of repeats)")
ax.tick_params(axis="x", rotation=30)
ax.grid(axis="y", alpha=0.3)
fig.tight_layout()
fig.savefig(out / "capacity.png", dpi=120)
plt.close(fig)
fig, ax = plt.subplots(figsize=(max(6, len(labels) * 1.1), 4.5))
width = 0.8 / len(COMPONENTS)
for i, comp in enumerate(COMPONENTS):
values = []
for key in configs:
at = [c["at"] for c in capacities if (c["series"], c["config"]) == key and c["at"]]
cpu = [float(a[f"cpu_{comp}_pct"]) for a in at if a[f"cpu_{comp}_pct"] != ""]
values.append(statistics.median(cpu) if cpu else 0)
ax.bar([j + i * width for j in range(len(configs))], values, width, label=comp)
ax.set_xticks([j + 0.4 - width / 2 for j in range(len(configs))], labels, rotation=30)
ax.set_ylabel("Mean CPU at capacity, % of one core")
ax.set_title("Which component is busy at capacity")
ax.grid(axis="y", alpha=0.3)
ax.legend()
fig.tight_layout()
fig.savefig(out / "cpu-at-capacity.png", dpi=120)
plt.close(fig)
fig, ax = plt.subplots(figsize=(7, 4.5))
for series, name in configs:
xs, ys, _ = median_by_offered(series, name, "pool_waits_per_s")
if xs:
ax.plot(xs, ys, marker="o", label=f"{series}/{name}")
ax.set_xlabel("Offered requests per second")
ax.set_ylabel("Waits for a pooled connection per second")
ax.set_title("Connection pool waits, all replicas")
ax.grid(alpha=0.3)
ax.legend()
fig.tight_layout()
fig.savefig(out / "pool-waits.png", dpi=120)
plt.close(fig)
print(f"{len(runs)} runs, {len(rows)} steps -> {out}")
if __name__ == "__main__":
if len(sys.argv) != 3:
sys.exit(__doc__)
main(pathlib.Path(sys.argv[1]), pathlib.Path(sys.argv[2]))