forked from mtusi2026/mtusi2026lab3
182 lines
7.6 KiB
Python
182 lines
7.6 KiB
Python
#!/usr/bin/env python3
|
|
"""Summarize independent repeats without treating them as one latency sample."""
|
|
|
|
import argparse
|
|
import csv
|
|
import json
|
|
import re
|
|
from collections import defaultdict
|
|
from pathlib import Path
|
|
from statistics import mean, median
|
|
|
|
import matplotlib
|
|
|
|
matplotlib.use("Agg")
|
|
import matplotlib.pyplot as plt # noqa: E402
|
|
|
|
|
|
def mib(value):
|
|
number, unit = re.fullmatch(r"([0-9.]+)(\w+)", value.strip()).groups()
|
|
return float(number) * {"B": 1 / 1048576, "KiB": 1 / 1024, "MiB": 1, "GiB": 1024}[unit]
|
|
|
|
|
|
def resources(trial):
|
|
cpu, memory = defaultdict(list), defaultdict(list)
|
|
for sample in trial["resource_samples"]:
|
|
per_role = defaultdict(float)
|
|
for container in sample.get("containers", []):
|
|
role = next(
|
|
(r for r in ("api", "postgres", "valkey", "lb") if f"-{r}-" in container["Name"]),
|
|
"generator",
|
|
)
|
|
per_role[role] += float(container["CPUPerc"].rstrip("%"))
|
|
memory[role].append(mib(container["MemUsage"].split("/")[0]))
|
|
for role, value in per_role.items():
|
|
cpu[role].append(value)
|
|
return {
|
|
role: {"cpu_percent": mean(cpu[role]), "max_container_mib": max(memory[role])}
|
|
for role in cpu
|
|
}
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("directory", type=Path)
|
|
args = parser.parse_args()
|
|
manifest = json.loads((args.directory / "manifest.json").read_text())
|
|
groups = defaultdict(list)
|
|
for path in sorted(args.directory.glob("trial-*.json")):
|
|
d = json.loads(path.read_text())
|
|
c = d["configuration"]
|
|
groups[
|
|
(c["series"], c["replicas"], c["concurrency"], c["optimized"], c.get("postgres_cpu", 1))
|
|
].append(d)
|
|
assert sum(map(len, groups.values())) == len(manifest["trials"])
|
|
rows = []
|
|
for (series, n, vus, optimized, pgcpu), trials in sorted(groups.items()):
|
|
assert len(trials) == 3, "A series needs three complete independent repeats"
|
|
resource = [resources(t) for t in trials]
|
|
c = trials[0]["configuration"]
|
|
row = {
|
|
"series": series,
|
|
"replicas": n,
|
|
"concurrency": vus,
|
|
"optimized": optimized,
|
|
"postgres_cpu": pgcpu,
|
|
"total_api_cpu": c["cpu_each"] * n,
|
|
"total_api_memory_mib": c["memory_mib_each"] * n,
|
|
"repeats": len(trials),
|
|
"requests": sum(t["requests"] for t in trials),
|
|
"errors": sum(t["errors"] for t in trials),
|
|
}
|
|
for metric in ("successful_rps", "p50_ms", "p95_ms", "p99_ms"):
|
|
values = [t[metric] for t in trials]
|
|
row[metric] = median(values)
|
|
row[metric + "_min"] = min(values)
|
|
row[metric + "_max"] = max(values)
|
|
row["begin_per_http_request"] = median(
|
|
sum(q["calls"] for q in t["sql_stats"] if q["query"].strip() == "BEGIN")
|
|
/ t["raw_k6"]["metrics"]["http_reqs"]["values"]["count"]
|
|
for t in trials
|
|
)
|
|
for role in ("api", "postgres", "lb", "valkey", "generator"):
|
|
row[role + "_cpu_percent"] = median(r[role]["cpu_percent"] for r in resource)
|
|
row[role + "_max_container_mib"] = max(r[role]["max_container_mib"] for r in resource)
|
|
row["api_quota_utilization_percent"] = row["api_cpu_percent"] / row["total_api_cpu"]
|
|
row["postgres_quota_utilization_percent"] = row["postgres_cpu_percent"] / pgcpu
|
|
rows.append(row)
|
|
(args.directory / "analysis.json").write_text(json.dumps(rows, indent=2) + "\n")
|
|
with (args.directory / "analysis.csv").open("w", newline="") as stream:
|
|
writer = csv.DictWriter(stream, fieldnames=list(rows[0]), lineterminator="\n")
|
|
writer.writeheader()
|
|
writer.writerows(rows)
|
|
lines = [
|
|
"| Series | API | VU | PG CPU | RPS median [min; max] | p50/p95/p99 ms | "
|
|
"Errors/requests | API CPU / PG CPU % | Max API MiB |",
|
|
"| --- | ---: | ---: | ---: | --- | --- | --- | --- | ---: |",
|
|
]
|
|
for r in rows:
|
|
lines.append(
|
|
f"| {r['series']} | {r['replicas']} | {r['concurrency']} | {r['postgres_cpu']} | "
|
|
f"{r['successful_rps']:.1f} [{r['successful_rps_min']:.1f}; "
|
|
f"{r['successful_rps_max']:.1f}] | "
|
|
f"{r['p50_ms']:.1f}/{r['p95_ms']:.1f}/{r['p99_ms']:.1f} | "
|
|
f"{r['errors']}/{r['requests']} | "
|
|
f"{r['api_cpu_percent']:.1f}/{r['postgres_cpu_percent']:.1f} | "
|
|
f"{r['api_max_container_mib']:.1f} |"
|
|
)
|
|
(args.directory / "results.md").write_text("\n".join(lines) + "\n")
|
|
plt.rcParams.update(
|
|
{
|
|
"font.size": 10,
|
|
"figure.dpi": 140,
|
|
"axes.grid": True,
|
|
"grid.alpha": 0.2,
|
|
"axes.spines.top": False,
|
|
"axes.spines.right": False,
|
|
}
|
|
)
|
|
fig, axes = plt.subplots(1, 3, figsize=(13, 4), constrained_layout=True)
|
|
for series, label in [
|
|
("fixed", "Fixed total: 2 CPU / 1 GiB"),
|
|
("growing", "Per replica: 1 CPU / 512 MiB"),
|
|
]:
|
|
values = [r for r in rows if r["series"] == series]
|
|
xs = [r["replicas"] for r in values]
|
|
for ax, metric, title in zip(
|
|
axes,
|
|
["successful_rps", "p95_ms", "api_quota_utilization_percent"],
|
|
["Successful requests/s", "p95 latency (ms)", "API CPU / quota (%)"],
|
|
strict=True,
|
|
):
|
|
ax.plot(xs, [r[metric] for r in values], marker="o", label=label)
|
|
if metric != "api_quota_utilization_percent":
|
|
ax.fill_between(
|
|
xs,
|
|
[r[metric + "_min"] for r in values],
|
|
[r[metric + "_max"] for r in values],
|
|
alpha=0.12,
|
|
)
|
|
ax.set(title=title, xlabel="API replicas", xticks=[1, 2, 4])
|
|
axes[0].legend(fontsize=8)
|
|
fig.suptitle("Scaling: same 90% read / 10% write workload, 32 VUs; median of 3 repeats")
|
|
fig.savefig(args.directory / "scaling.png")
|
|
plt.close(fig)
|
|
baseline = next(r for r in rows if r["series"] == "fixed" and r["replicas"] == 2)
|
|
optimized = next(r for r in rows if r["series"] == "improvement")
|
|
values = sorted(
|
|
[baseline] + [r for r in rows if r["series"] == "saturation"],
|
|
key=lambda r: r["concurrency"],
|
|
)
|
|
fig, axes = plt.subplots(1, 3, figsize=(13, 4), constrained_layout=True)
|
|
xs = [r["concurrency"] for r in values]
|
|
axes[0].plot(xs, [r["successful_rps"] for r in values], "o-", label="RPS")
|
|
tail = axes[0].twinx()
|
|
tail.plot(xs, [r["p95_ms"] for r in values], "s--", color="tab:orange", label="p95 ms")
|
|
tail.set_ylabel("p95 ms", color="tab:orange")
|
|
axes[0].set(title="Concurrency, baseline", xlabel="VUs", ylabel="Successful RPS")
|
|
db = sorted(
|
|
[optimized] + [r for r in rows if r["series"] == "database"],
|
|
key=lambda r: r["postgres_cpu"],
|
|
)
|
|
axes[1].plot([r["postgres_cpu"] for r in db], [r["successful_rps"] for r in db], "o-")
|
|
axes[1].set(
|
|
title="Controlled PostgreSQL saturation", xlabel="PostgreSQL CPU quota", ylabel="RPS"
|
|
)
|
|
axes[2].bar(
|
|
["Baseline", "Autocommit\nsingle SELECT"],
|
|
[baseline["successful_rps"], optimized["successful_rps"]],
|
|
color=["tab:blue", "tab:green"],
|
|
)
|
|
axes[2].set(title="Same 2 API / 2 CPU / 32 VUs", ylabel="Successful RPS")
|
|
gain = (optimized["successful_rps"] / baseline["successful_rps"] - 1) * 100
|
|
fig.suptitle(f"Bottlenecks and one change: median RPS gain {gain:+.1f}%")
|
|
fig.savefig(args.directory / "bottlenecks.png")
|
|
plt.close(fig)
|
|
print("\n".join(lines))
|
|
print(f"Optimization median gain: {gain:+.2f}%")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|