167 lines
6.8 KiB
Python
167 lines
6.8 KiB
Python
"""Summarize successes separately from offered/attempted requests and sum replica resources."""
|
|
|
|
import json
|
|
import re
|
|
import statistics
|
|
import sys
|
|
from collections import defaultdict
|
|
from pathlib import Path
|
|
|
|
|
|
def memory_bytes(text):
|
|
value, unit = re.fullmatch(r"([0-9.]+)([A-Za-z]+)", text.strip()).groups()
|
|
factors = {
|
|
"B": 1,
|
|
"KiB": 1024,
|
|
"MiB": 1024**2,
|
|
"GiB": 1024**3,
|
|
"kB": 1000,
|
|
"MB": 1000**2,
|
|
"GB": 1000**3,
|
|
}
|
|
return float(value) * factors[unit]
|
|
|
|
|
|
def resource_summary(run):
|
|
cpu, ram, db_cpu, lb_cpu = [], [], [], []
|
|
by_id = {c["id"][:12]: c["service"] for c in run["actual_containers"]}
|
|
for sample in run["resource_samples"]:
|
|
api_cpu = api_ram = 0
|
|
for row in sample["containers"]:
|
|
role = by_id.get(row["ID"][:12])
|
|
percent = float(row["CPUPerc"].rstrip("%"))
|
|
if role == "api":
|
|
api_cpu += percent
|
|
api_ram += memory_bytes(row["MemUsage"].split("/")[0].strip())
|
|
elif role == "postgres":
|
|
db_cpu.append(percent)
|
|
elif role == "lb":
|
|
lb_cpu.append(percent)
|
|
cpu.append(api_cpu)
|
|
ram.append(api_ram / 1024**2)
|
|
return {
|
|
"mean_api_cpu_percent_sum": statistics.mean(cpu) if cpu else None,
|
|
"max_api_memory_mib_sum": max(ram, default=0),
|
|
"mean_db_cpu_percent": statistics.mean(db_cpu) if db_cpu else None,
|
|
"mean_lb_cpu_percent": statistics.mean(lb_cpu) if lb_cpu else None,
|
|
}
|
|
|
|
|
|
def main():
|
|
source = Path(sys.argv[1])
|
|
data = json.loads(source.read_text())
|
|
groups = defaultdict(list)
|
|
for run in data["runs"]:
|
|
assert run["scheduled"] == run["attempted"] + run["dropped"]
|
|
assert run["attempted"] == run["successful"] + sum(run["errors"].values())
|
|
run["resources"] = resource_summary(run)
|
|
groups[run["configuration"], run["offered_rate"]].append(run)
|
|
rows = []
|
|
for (name, rate), runs in groups.items():
|
|
row = {
|
|
"configuration": name,
|
|
"offered_rate": rate,
|
|
"repeats": len(runs),
|
|
"median_successful_rps": statistics.median(r["successful_rps"] for r in runs),
|
|
"median_p95_ms": statistics.median(r["latency_success_ms"]["p95"] for r in runs),
|
|
"total_successful": sum(r["successful"] for r in runs),
|
|
"total_errors": sum(sum(r["errors"].values()) for r in runs),
|
|
"total_dropped": sum(r["dropped"] for r in runs),
|
|
"total_scheduler_dropped": sum(r["dropped_scheduler"] for r in runs),
|
|
"max_driver_cpu_fraction": max(r["driver_cpu_fraction_one_core"] for r in runs),
|
|
"total_scheduled": sum(r["scheduled"] for r in runs),
|
|
"median_db_cpu_percent": statistics.median(
|
|
r["resources"]["mean_db_cpu_percent"] for r in runs
|
|
),
|
|
"median_api_cpu_percent_sum": statistics.median(
|
|
r["resources"]["mean_api_cpu_percent_sum"] for r in runs
|
|
),
|
|
"max_api_memory_mib_sum": max(r["resources"]["max_api_memory_mib_sum"] for r in runs),
|
|
}
|
|
rows.append(row)
|
|
dest = source.with_name(source.stem + "-summary.json")
|
|
dest.write_text(json.dumps(rows, indent=2) + "\n")
|
|
text = [
|
|
"# Scaling results",
|
|
"",
|
|
"CPU%: 100% means one logical CPU. Memory is summed Docker MemUsage, not heap RSS.",
|
|
"",
|
|
"|Configuration|Offered/s|Success/s median|p95 ms median|Errors total|Dropped total|DB CPU% median|API CPU% sum median|API peak MiB sum|",
|
|
"|---|---:|---:|---:|---:|---:|---:|---:|---:|",
|
|
]
|
|
for row in rows:
|
|
text.append(
|
|
"|{configuration}|{offered_rate}|{median_successful_rps:.2f}|{median_p95_ms:.2f}|{total_errors}|{total_dropped}|{median_db_cpu_percent:.2f}|{median_api_cpu_percent_sum:.2f}|{max_api_memory_mib_sum:.2f}|".format(
|
|
**row
|
|
)
|
|
)
|
|
source.with_name(source.stem + "-summary.md").write_text("\n".join(text) + "\n")
|
|
import matplotlib
|
|
|
|
matplotlib.use("Agg")
|
|
import matplotlib.pyplot as plt
|
|
|
|
fig, axes = plt.subplots(1, 3, figsize=(15, 4), constrained_layout=True)
|
|
for ax, series in zip(axes[:2], ["fixed", "growth"], strict=True):
|
|
for name in [f"{series}-{n}" for n in [1, 2, 4]]:
|
|
points = sorted(
|
|
[r for r in rows if r["configuration"] == name], key=lambda r: r["offered_rate"]
|
|
)
|
|
ax.plot(
|
|
[p["offered_rate"] for p in points],
|
|
[p["median_successful_rps"] for p in points],
|
|
"o-",
|
|
label=name,
|
|
)
|
|
ax.set(
|
|
title=series + " API budgets",
|
|
xlabel="Offered requests/s",
|
|
ylabel="Successful requests/s",
|
|
)
|
|
ax.grid(alpha=0.3)
|
|
ax.legend()
|
|
for name in ["growth-4", "db-improved"]:
|
|
points = sorted(
|
|
[r for r in rows if r["configuration"] == name], key=lambda r: r["offered_rate"]
|
|
)
|
|
axes[2].plot(
|
|
[p["offered_rate"] for p in points],
|
|
[p["median_successful_rps"] for p in points],
|
|
"o-",
|
|
label=name,
|
|
)
|
|
axes[2].set(
|
|
title="Controlled PostgreSQL CPU change",
|
|
xlabel="Offered requests/s",
|
|
ylabel="Successful requests/s",
|
|
)
|
|
axes[2].grid(alpha=0.3)
|
|
axes[2].legend()
|
|
fig.savefig(source.with_name(source.stem + ".svg"))
|
|
fig.savefig(source.with_name(source.stem + ".png"), dpi=130)
|
|
plt.close(fig)
|
|
fig, axes = plt.subplots(1, 3, figsize=(15, 4), constrained_layout=True)
|
|
for name in ["growth-4", "db-improved"]:
|
|
points = sorted([r for r in rows if r["configuration"] == name], key=lambda r: r["offered_rate"])
|
|
xs = [p["offered_rate"] for p in points]
|
|
axes[0].plot(xs, [p["median_db_cpu_percent"] for p in points], "o-", label=name)
|
|
axes[1].plot(xs, [p["max_driver_cpu_fraction"] * 100 for p in points], "o-", label=name)
|
|
axes[2].plot(xs, [p["total_dropped"] / p["total_scheduled"] * 100 for p in points], "o-", label=name)
|
|
axes[0].axhline(25, linestyle="--", color="gray", label="baseline quota25%")
|
|
axes[0].axhline(50, linestyle=":", color="gray", label="improved quota50%")
|
|
axes[0].set(title="PostgreSQL CPU", ylabel="CPU% (100%=one core)")
|
|
axes[1].set(title="Driver process CPU, maximum repeat", ylabel="CPU% (100%=one core)")
|
|
axes[2].set(title="Dropped scheduling slots", ylabel="Dropped / scheduled %")
|
|
for ax in axes:
|
|
ax.set_xlabel("Offered requests/s")
|
|
ax.grid(alpha=.3)
|
|
ax.legend()
|
|
fig.savefig(source.with_name(source.stem + "-resources.svg"))
|
|
fig.savefig(source.with_name(source.stem + "-resources.png"), dpi=130)
|
|
plt.close(fig)
|
|
print("\n".join(text))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|