Files
mtusi2026lab3/scripts/plot-scaling-diagnostics.py
T
qrtx c1fec73ccc perf(bench): add scaling experiments and saturation diagnostics
Compare API and PostgreSQL CPU budgets and connection pool limits. Collect repeatable measurements and generate report aggregates and graphs.
2026-10-29 11:49:51 +03:00

160 lines
9.2 KiB
Python

#!/usr/bin/env python3
"""Plot completed diagnostic runs using matplotlib; reads results.csv and manifest.json."""
import argparse
import csv
import json
from pathlib import Path
import statistics
import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("directory", type=Path)
parser.add_argument("--phase", help="Only plot this CSV phase, e.g. confirmation")
parser.add_argument("--all-latency", action="store_true", help="Use all-request p95, including failures")
parser.add_argument("--valid-delivery-only", action="store_true", help="Exclude runs with unverified arrival rate")
args = parser.parse_args()
manifest = json.loads((args.directory / "manifest.json").read_text(encoding="utf-8"))
with (args.directory / "results.csv").open(encoding="utf-8") as stream:
rows = list(csv.DictReader(stream))
if args.phase:
rows = [row for row in rows if row.get("phase") == args.phase]
if args.valid_delivery_only:
rows = [row for row in rows if row.get("delivery_valid") in ("True", "true", "1")]
if not rows:
parser.error("no results")
for row in rows:
row['configuration'] = row.get('configuration') or next(
(profile for profile in manifest.get('profiles', []) if row['label'].startswith(profile + '-r')),
str(row['replicas']))
for key in ("replicas", "offered_rps"):
row[key] = int(row[key])
for key in ("successful_rps", "attempted_rps", "p95_ms", "drop_percent", "acquisition_mean_ms",
"pool_max_waiting", "api_cpu_cores", "api_max_throttle_percent",
"pg_cpu_cores", "pg_throttle_percent"):
row[key] = float(row[key] or 'nan')
if args.all_latency:
if not row.get("all_request_p95_ms"):
parser.error("all-request latency is missing from this dataset")
row["p95_ms"] = float(row["all_request_p95_ms"])
summary = json.loads((args.directory / row["label"] / "summary.json").read_text(encoding="utf-8"))
m = summary["measurement"]["all"]
row["error_percent"] = 100*m["errors"]/max(m["attempted"], 1)
pg = next(c for group in manifest["containers"].values() for c in group if "-postgres-" in c["name"])
pg_cpus = pg["nano_cpus"]/1e9
memory_mib = pg["memory_bytes"]/1024**2
configurations = sorted({r['configuration'] for r in rows})
palette = ['#2563eb', '#059669', '#d97706', '#9333ea', '#db2777', '#0891b2', '#64748b']
colors = {configuration: palette[i % len(palette)] for i, configuration in enumerate(configurations)}
budgets = {}
labels = {}
for configuration in configurations:
apis = [c for c in manifest.get('containers', {}).get(configuration, []) if '-api-' in c['name']]
replicas = next(r['replicas'] for r in rows if r['configuration'] == configuration)
if apis:
cpu = sum(c['nano_cpus'] for c in apis) / 1e9
memory = sum(c['memory_bytes'] for c in apis) / 1024**3
budgets[configuration] = (cpu, memory)
labels[configuration] = f'{replicas} API ({cpu:g} CPU / {memory:g} GiB)'
else:
labels[configuration] = f'{configuration}: {replicas} API'
budget_description = '; '.join(f'{cpu:g} CPU / {memory:g} GiB'
for cpu, memory in sorted(set(budgets.values()))) or 'see manifest'
def draw(ax, field, title, ylabel, log=False):
for configuration in configurations:
selected = [r for r in rows if r['configuration'] == configuration]
rates = sorted({r["offered_rps"] for r in selected})
samples = [[r[field] for r in selected if r["offered_rps"] == rate] for rate in rates]
values = [statistics.median(v) for v in samples]
ax.plot(rates, values, marker="o", color=colors[configuration], label=labels[configuration])
ax.fill_between(rates, [min(v) for v in samples], [max(v) for v in samples],
color=colors[configuration], alpha=0.12)
ax.set(title=title, xlabel="Scheduled requests/s", ylabel=ylabel)
ax.grid(alpha=0.25)
if log:
ax.set_yscale("log")
else:
ax.set_ylim(bottom=0)
def save(fig, name, caption):
if args.valid_delivery_only:
caption += "\nUnderfed runs excluded; sample counts can differ between points."
fig.text(0.5, 0.015, caption, ha="center", fontsize=9)
fig.tight_layout(rect=(0, 0.055, 1, 0.94))
for extension in ("png", "svg"):
fig.savefig(args.directory / f"{name}.{extension}", dpi=160, bbox_inches="tight", pad_inches=0.25)
plt.close(fig)
plt.rcParams.update({"font.size": 10, "axes.titlesize": 11})
fig, axes = plt.subplots(2, 2, figsize=(12, 8))
fig.suptitle(f"PostgreSQL {pg_cpus:g} CPU / {memory_mib:g} MiB; API budgets: {budget_description}", fontsize=12)
draw(axes[0, 0], "successful_rps", "Successful throughput", "Successful requests/s")
rates = sorted({r["offered_rps"] for r in rows})
axes[0, 0].plot(rates, rates, color="#475569", linestyle="--", alpha=0.7, label="Scheduled")
axes[0, 0].set_ylim(0, max(axes[0, 0].get_ylim()[1], max(rates)*1.05))
axes[0, 0].legend()
draw(axes[0, 1], "p95_ms", "Latency of all requests" if args.all_latency else "Latency of successful requests", "p95, ms (log scale)", log=True)
if args.all_latency:
axes[0, 1].axhline(1000, color="#64748b", linestyle="--", linewidth=1, label="1000 ms criterion")
axes[0, 1].legend(fontsize=8)
draw(axes[1, 0], "error_percent", "Errors among attempted requests", "Errors, %")
axes[1, 0].axhline(1, color="#64748b", linestyle="--", linewidth=1)
axes[1, 0].set_ylim(0, max(axes[1, 0].get_ylim()[1], 1.05))
draw(axes[1, 1], "drop_percent", "Slots dropped by client", "Dropped scheduled slots, %")
drop_limit = 2 if args.valid_delivery_only else 5
axes[1, 1].axhline(drop_limit, color="#64748b", linestyle="--", linewidth=1)
axes[1, 1].set_ylim(0, max(axes[1, 1].get_ylim()[1], drop_limit*1.05))
save(fig, "performance", "Lines: medians of repeats; shading: min-max. High client-drop points do not establish server capacity.")
fig, axes = plt.subplots(2, 3, figsize=(15, 8))
fig.suptitle("Resource and connection-pool diagnostics", fontsize=14)
draw(axes[0, 0], "pg_cpu_cores", "PostgreSQL CPU usage", "CPU cores")
axes[0, 0].axhline(pg_cpus, color="#64748b", linestyle="--", linewidth=1, label="CPU limit")
axes[0, 0].set_ylim(0, max(axes[0, 0].get_ylim()[1], pg_cpus*1.05))
draw(axes[0, 1], "api_cpu_cores", "Total API CPU usage", "CPU cores")
for configuration, (cpu, _) in budgets.items():
axes[0, 1].axhline(cpu, color=colors[configuration], linestyle='--', linewidth=1,
label=f'{configuration} limit: {cpu:g} CPU')
if budgets:
axes[0, 1].set_ylim(0, max(axes[0, 1].get_ylim()[1], max(cpu for cpu, _ in budgets.values()) * 1.05))
axes[0, 1].legend(fontsize=8)
draw(axes[0, 2], "pg_throttle_percent", "PostgreSQL CPU throttling", "Throttled periods, %")
draw(axes[1, 0], "api_max_throttle_percent", "Most throttled API replica", "Throttled periods, %")
draw(axes[1, 1], "acquisition_mean_ms", "Mean connection acquisition", "Acquisition, ms (log scale)", log=True)
draw(axes[1, 2], "pool_max_waiting", "Largest observed pool queue", "Waiting requests")
axes[0, 0].legend()
save(fig, "resources", "Lines: medians of repeats; shading: min-max. Queue sampled every 100 ms; acquisition includes connection setup.")
fig, axes = plt.subplots(1, 2, figsize=(12, 5))
fig.suptitle("Latency and errors against the workload actually delivered", fontsize=14)
for configuration in configurations:
selected = [r for r in rows if r['configuration'] == configuration]
rates = sorted({r["offered_rps"] for r in selected})
groups = [[r for r in selected if r["offered_rps"] == rate] for rate in rates]
x = [statistics.median(r["attempted_rps"] for r in group) for group in groups]
for axis, field, title, ylabel in (
(axes[0], "p95_ms", "Latency", "p95, ms (log scale)"),
(axes[1], "error_percent", "Errors", "Errors among attempts, %"),
):
y = [statistics.median(r[field] for r in group) for group in groups]
axis.plot(x, y, marker="o", color=colors[configuration], label=labels[configuration])
for rate, xp, yp in zip(rates, x, y):
axis.annotate(str(rate), (xp, yp), xytext=(4, 4), textcoords="offset points", fontsize=8)
axis.set(title=title, xlabel="Actually attempted requests/s", ylabel=ylabel)
axis.grid(alpha=0.25)
axes[0].set_yscale("log")
if args.all_latency:
axes[0].axhline(1000, color="#64748b", linestyle="--", linewidth=1, label="1000 ms criterion")
axes[1].set_ylim(bottom=0)
axes[0].legend()
save(fig, "delivered-load", "Points: medians of repeats. Small labels: scheduled RPS. Client shares the Docker VM; no server-capacity claim.")
print(f"Wrote performance/resources/delivered-load PNG and SVG for {len(rows)} runs to {args.directory}")
if __name__ == "__main__":
main()