forked from mtusi2026/mtusi2026lab3
perf(bench): add scaling experiments and saturation diagnostics
Compare API and PostgreSQL CPU budgets and connection pool limits. Collect repeatable measurements and generate report aggregates and graphs.
This commit is contained in:
@@ -0,0 +1,159 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Plot completed diagnostic runs using matplotlib; reads results.csv and manifest.json."""
|
||||
import argparse
|
||||
import csv
|
||||
import json
|
||||
from pathlib import Path
|
||||
import statistics
|
||||
|
||||
import matplotlib
|
||||
matplotlib.use("Agg")
|
||||
import matplotlib.pyplot as plt
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("directory", type=Path)
|
||||
parser.add_argument("--phase", help="Only plot this CSV phase, e.g. confirmation")
|
||||
parser.add_argument("--all-latency", action="store_true", help="Use all-request p95, including failures")
|
||||
parser.add_argument("--valid-delivery-only", action="store_true", help="Exclude runs with unverified arrival rate")
|
||||
args = parser.parse_args()
|
||||
manifest = json.loads((args.directory / "manifest.json").read_text(encoding="utf-8"))
|
||||
with (args.directory / "results.csv").open(encoding="utf-8") as stream:
|
||||
rows = list(csv.DictReader(stream))
|
||||
if args.phase:
|
||||
rows = [row for row in rows if row.get("phase") == args.phase]
|
||||
if args.valid_delivery_only:
|
||||
rows = [row for row in rows if row.get("delivery_valid") in ("True", "true", "1")]
|
||||
if not rows:
|
||||
parser.error("no results")
|
||||
for row in rows:
|
||||
row['configuration'] = row.get('configuration') or next(
|
||||
(profile for profile in manifest.get('profiles', []) if row['label'].startswith(profile + '-r')),
|
||||
str(row['replicas']))
|
||||
for key in ("replicas", "offered_rps"):
|
||||
row[key] = int(row[key])
|
||||
for key in ("successful_rps", "attempted_rps", "p95_ms", "drop_percent", "acquisition_mean_ms",
|
||||
"pool_max_waiting", "api_cpu_cores", "api_max_throttle_percent",
|
||||
"pg_cpu_cores", "pg_throttle_percent"):
|
||||
row[key] = float(row[key] or 'nan')
|
||||
if args.all_latency:
|
||||
if not row.get("all_request_p95_ms"):
|
||||
parser.error("all-request latency is missing from this dataset")
|
||||
row["p95_ms"] = float(row["all_request_p95_ms"])
|
||||
summary = json.loads((args.directory / row["label"] / "summary.json").read_text(encoding="utf-8"))
|
||||
m = summary["measurement"]["all"]
|
||||
row["error_percent"] = 100*m["errors"]/max(m["attempted"], 1)
|
||||
pg = next(c for group in manifest["containers"].values() for c in group if "-postgres-" in c["name"])
|
||||
pg_cpus = pg["nano_cpus"]/1e9
|
||||
memory_mib = pg["memory_bytes"]/1024**2
|
||||
configurations = sorted({r['configuration'] for r in rows})
|
||||
palette = ['#2563eb', '#059669', '#d97706', '#9333ea', '#db2777', '#0891b2', '#64748b']
|
||||
colors = {configuration: palette[i % len(palette)] for i, configuration in enumerate(configurations)}
|
||||
budgets = {}
|
||||
labels = {}
|
||||
for configuration in configurations:
|
||||
apis = [c for c in manifest.get('containers', {}).get(configuration, []) if '-api-' in c['name']]
|
||||
replicas = next(r['replicas'] for r in rows if r['configuration'] == configuration)
|
||||
if apis:
|
||||
cpu = sum(c['nano_cpus'] for c in apis) / 1e9
|
||||
memory = sum(c['memory_bytes'] for c in apis) / 1024**3
|
||||
budgets[configuration] = (cpu, memory)
|
||||
labels[configuration] = f'{replicas} API ({cpu:g} CPU / {memory:g} GiB)'
|
||||
else:
|
||||
labels[configuration] = f'{configuration}: {replicas} API'
|
||||
budget_description = '; '.join(f'{cpu:g} CPU / {memory:g} GiB'
|
||||
for cpu, memory in sorted(set(budgets.values()))) or 'see manifest'
|
||||
|
||||
def draw(ax, field, title, ylabel, log=False):
|
||||
for configuration in configurations:
|
||||
selected = [r for r in rows if r['configuration'] == configuration]
|
||||
rates = sorted({r["offered_rps"] for r in selected})
|
||||
samples = [[r[field] for r in selected if r["offered_rps"] == rate] for rate in rates]
|
||||
values = [statistics.median(v) for v in samples]
|
||||
ax.plot(rates, values, marker="o", color=colors[configuration], label=labels[configuration])
|
||||
ax.fill_between(rates, [min(v) for v in samples], [max(v) for v in samples],
|
||||
color=colors[configuration], alpha=0.12)
|
||||
ax.set(title=title, xlabel="Scheduled requests/s", ylabel=ylabel)
|
||||
ax.grid(alpha=0.25)
|
||||
if log:
|
||||
ax.set_yscale("log")
|
||||
else:
|
||||
ax.set_ylim(bottom=0)
|
||||
|
||||
def save(fig, name, caption):
|
||||
if args.valid_delivery_only:
|
||||
caption += "\nUnderfed runs excluded; sample counts can differ between points."
|
||||
fig.text(0.5, 0.015, caption, ha="center", fontsize=9)
|
||||
fig.tight_layout(rect=(0, 0.055, 1, 0.94))
|
||||
for extension in ("png", "svg"):
|
||||
fig.savefig(args.directory / f"{name}.{extension}", dpi=160, bbox_inches="tight", pad_inches=0.25)
|
||||
plt.close(fig)
|
||||
|
||||
plt.rcParams.update({"font.size": 10, "axes.titlesize": 11})
|
||||
fig, axes = plt.subplots(2, 2, figsize=(12, 8))
|
||||
fig.suptitle(f"PostgreSQL {pg_cpus:g} CPU / {memory_mib:g} MiB; API budgets: {budget_description}", fontsize=12)
|
||||
draw(axes[0, 0], "successful_rps", "Successful throughput", "Successful requests/s")
|
||||
rates = sorted({r["offered_rps"] for r in rows})
|
||||
axes[0, 0].plot(rates, rates, color="#475569", linestyle="--", alpha=0.7, label="Scheduled")
|
||||
axes[0, 0].set_ylim(0, max(axes[0, 0].get_ylim()[1], max(rates)*1.05))
|
||||
axes[0, 0].legend()
|
||||
draw(axes[0, 1], "p95_ms", "Latency of all requests" if args.all_latency else "Latency of successful requests", "p95, ms (log scale)", log=True)
|
||||
if args.all_latency:
|
||||
axes[0, 1].axhline(1000, color="#64748b", linestyle="--", linewidth=1, label="1000 ms criterion")
|
||||
axes[0, 1].legend(fontsize=8)
|
||||
draw(axes[1, 0], "error_percent", "Errors among attempted requests", "Errors, %")
|
||||
axes[1, 0].axhline(1, color="#64748b", linestyle="--", linewidth=1)
|
||||
axes[1, 0].set_ylim(0, max(axes[1, 0].get_ylim()[1], 1.05))
|
||||
draw(axes[1, 1], "drop_percent", "Slots dropped by client", "Dropped scheduled slots, %")
|
||||
drop_limit = 2 if args.valid_delivery_only else 5
|
||||
axes[1, 1].axhline(drop_limit, color="#64748b", linestyle="--", linewidth=1)
|
||||
axes[1, 1].set_ylim(0, max(axes[1, 1].get_ylim()[1], drop_limit*1.05))
|
||||
save(fig, "performance", "Lines: medians of repeats; shading: min-max. High client-drop points do not establish server capacity.")
|
||||
|
||||
fig, axes = plt.subplots(2, 3, figsize=(15, 8))
|
||||
fig.suptitle("Resource and connection-pool diagnostics", fontsize=14)
|
||||
draw(axes[0, 0], "pg_cpu_cores", "PostgreSQL CPU usage", "CPU cores")
|
||||
axes[0, 0].axhline(pg_cpus, color="#64748b", linestyle="--", linewidth=1, label="CPU limit")
|
||||
axes[0, 0].set_ylim(0, max(axes[0, 0].get_ylim()[1], pg_cpus*1.05))
|
||||
draw(axes[0, 1], "api_cpu_cores", "Total API CPU usage", "CPU cores")
|
||||
for configuration, (cpu, _) in budgets.items():
|
||||
axes[0, 1].axhline(cpu, color=colors[configuration], linestyle='--', linewidth=1,
|
||||
label=f'{configuration} limit: {cpu:g} CPU')
|
||||
if budgets:
|
||||
axes[0, 1].set_ylim(0, max(axes[0, 1].get_ylim()[1], max(cpu for cpu, _ in budgets.values()) * 1.05))
|
||||
axes[0, 1].legend(fontsize=8)
|
||||
draw(axes[0, 2], "pg_throttle_percent", "PostgreSQL CPU throttling", "Throttled periods, %")
|
||||
draw(axes[1, 0], "api_max_throttle_percent", "Most throttled API replica", "Throttled periods, %")
|
||||
draw(axes[1, 1], "acquisition_mean_ms", "Mean connection acquisition", "Acquisition, ms (log scale)", log=True)
|
||||
draw(axes[1, 2], "pool_max_waiting", "Largest observed pool queue", "Waiting requests")
|
||||
axes[0, 0].legend()
|
||||
save(fig, "resources", "Lines: medians of repeats; shading: min-max. Queue sampled every 100 ms; acquisition includes connection setup.")
|
||||
fig, axes = plt.subplots(1, 2, figsize=(12, 5))
|
||||
fig.suptitle("Latency and errors against the workload actually delivered", fontsize=14)
|
||||
for configuration in configurations:
|
||||
selected = [r for r in rows if r['configuration'] == configuration]
|
||||
rates = sorted({r["offered_rps"] for r in selected})
|
||||
groups = [[r for r in selected if r["offered_rps"] == rate] for rate in rates]
|
||||
x = [statistics.median(r["attempted_rps"] for r in group) for group in groups]
|
||||
for axis, field, title, ylabel in (
|
||||
(axes[0], "p95_ms", "Latency", "p95, ms (log scale)"),
|
||||
(axes[1], "error_percent", "Errors", "Errors among attempts, %"),
|
||||
):
|
||||
y = [statistics.median(r[field] for r in group) for group in groups]
|
||||
axis.plot(x, y, marker="o", color=colors[configuration], label=labels[configuration])
|
||||
for rate, xp, yp in zip(rates, x, y):
|
||||
axis.annotate(str(rate), (xp, yp), xytext=(4, 4), textcoords="offset points", fontsize=8)
|
||||
axis.set(title=title, xlabel="Actually attempted requests/s", ylabel=ylabel)
|
||||
axis.grid(alpha=0.25)
|
||||
axes[0].set_yscale("log")
|
||||
if args.all_latency:
|
||||
axes[0].axhline(1000, color="#64748b", linestyle="--", linewidth=1, label="1000 ms criterion")
|
||||
axes[1].set_ylim(bottom=0)
|
||||
axes[0].legend()
|
||||
save(fig, "delivered-load", "Points: medians of repeats. Small labels: scheduled RPS. Client shares the Docker VM; no server-capacity claim.")
|
||||
print(f"Wrote performance/resources/delivered-load PNG and SVG for {len(rows)} runs to {args.directory}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user