Files
mtusi2026lab3/research/analyze.py
T

182 lines
7.6 KiB
Python

#!/usr/bin/env python3
"""Summarize independent repeats without treating them as one latency sample."""
import argparse
import csv
import json
import re
from collections import defaultdict
from pathlib import Path
from statistics import mean, median
import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt # noqa: E402
def mib(value):
number, unit = re.fullmatch(r"([0-9.]+)(\w+)", value.strip()).groups()
return float(number) * {"B": 1 / 1048576, "KiB": 1 / 1024, "MiB": 1, "GiB": 1024}[unit]
def resources(trial):
cpu, memory = defaultdict(list), defaultdict(list)
for sample in trial["resource_samples"]:
per_role = defaultdict(float)
for container in sample.get("containers", []):
role = next(
(r for r in ("api", "postgres", "valkey", "lb") if f"-{r}-" in container["Name"]),
"generator",
)
per_role[role] += float(container["CPUPerc"].rstrip("%"))
memory[role].append(mib(container["MemUsage"].split("/")[0]))
for role, value in per_role.items():
cpu[role].append(value)
return {
role: {"cpu_percent": mean(cpu[role]), "max_container_mib": max(memory[role])}
for role in cpu
}
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("directory", type=Path)
args = parser.parse_args()
manifest = json.loads((args.directory / "manifest.json").read_text())
groups = defaultdict(list)
for path in sorted(args.directory.glob("trial-*.json")):
d = json.loads(path.read_text())
c = d["configuration"]
groups[
(c["series"], c["replicas"], c["concurrency"], c["optimized"], c.get("postgres_cpu", 1))
].append(d)
assert sum(map(len, groups.values())) == len(manifest["trials"])
rows = []
for (series, n, vus, optimized, pgcpu), trials in sorted(groups.items()):
assert len(trials) == 3, "A series needs three complete independent repeats"
resource = [resources(t) for t in trials]
c = trials[0]["configuration"]
row = {
"series": series,
"replicas": n,
"concurrency": vus,
"optimized": optimized,
"postgres_cpu": pgcpu,
"total_api_cpu": c["cpu_each"] * n,
"total_api_memory_mib": c["memory_mib_each"] * n,
"repeats": len(trials),
"requests": sum(t["requests"] for t in trials),
"errors": sum(t["errors"] for t in trials),
}
for metric in ("successful_rps", "p50_ms", "p95_ms", "p99_ms"):
values = [t[metric] for t in trials]
row[metric] = median(values)
row[metric + "_min"] = min(values)
row[metric + "_max"] = max(values)
row["begin_per_http_request"] = median(
sum(q["calls"] for q in t["sql_stats"] if q["query"].strip() == "BEGIN")
/ t["raw_k6"]["metrics"]["http_reqs"]["values"]["count"]
for t in trials
)
for role in ("api", "postgres", "lb", "valkey", "generator"):
row[role + "_cpu_percent"] = median(r[role]["cpu_percent"] for r in resource)
row[role + "_max_container_mib"] = max(r[role]["max_container_mib"] for r in resource)
row["api_quota_utilization_percent"] = row["api_cpu_percent"] / row["total_api_cpu"]
row["postgres_quota_utilization_percent"] = row["postgres_cpu_percent"] / pgcpu
rows.append(row)
(args.directory / "analysis.json").write_text(json.dumps(rows, indent=2) + "\n")
with (args.directory / "analysis.csv").open("w", newline="") as stream:
writer = csv.DictWriter(stream, fieldnames=list(rows[0]), lineterminator="\n")
writer.writeheader()
writer.writerows(rows)
lines = [
"| Series | API | VU | PG CPU | RPS median [min; max] | p50/p95/p99 ms | "
"Errors/requests | API CPU / PG CPU % | Max API MiB |",
"| --- | ---: | ---: | ---: | --- | --- | --- | --- | ---: |",
]
for r in rows:
lines.append(
f"| {r['series']} | {r['replicas']} | {r['concurrency']} | {r['postgres_cpu']} | "
f"{r['successful_rps']:.1f} [{r['successful_rps_min']:.1f}; "
f"{r['successful_rps_max']:.1f}] | "
f"{r['p50_ms']:.1f}/{r['p95_ms']:.1f}/{r['p99_ms']:.1f} | "
f"{r['errors']}/{r['requests']} | "
f"{r['api_cpu_percent']:.1f}/{r['postgres_cpu_percent']:.1f} | "
f"{r['api_max_container_mib']:.1f} |"
)
(args.directory / "results.md").write_text("\n".join(lines) + "\n")
plt.rcParams.update(
{
"font.size": 10,
"figure.dpi": 140,
"axes.grid": True,
"grid.alpha": 0.2,
"axes.spines.top": False,
"axes.spines.right": False,
}
)
fig, axes = plt.subplots(1, 3, figsize=(13, 4), constrained_layout=True)
for series, label in [
("fixed", "Fixed total: 2 CPU / 1 GiB"),
("growing", "Per replica: 1 CPU / 512 MiB"),
]:
values = [r for r in rows if r["series"] == series]
xs = [r["replicas"] for r in values]
for ax, metric, title in zip(
axes,
["successful_rps", "p95_ms", "api_quota_utilization_percent"],
["Successful requests/s", "p95 latency (ms)", "API CPU / quota (%)"],
strict=True,
):
ax.plot(xs, [r[metric] for r in values], marker="o", label=label)
if metric != "api_quota_utilization_percent":
ax.fill_between(
xs,
[r[metric + "_min"] for r in values],
[r[metric + "_max"] for r in values],
alpha=0.12,
)
ax.set(title=title, xlabel="API replicas", xticks=[1, 2, 4])
axes[0].legend(fontsize=8)
fig.suptitle("Scaling: same 90% read / 10% write workload, 32 VUs; median of 3 repeats")
fig.savefig(args.directory / "scaling.png")
plt.close(fig)
baseline = next(r for r in rows if r["series"] == "fixed" and r["replicas"] == 2)
optimized = next(r for r in rows if r["series"] == "improvement")
values = sorted(
[baseline] + [r for r in rows if r["series"] == "saturation"],
key=lambda r: r["concurrency"],
)
fig, axes = plt.subplots(1, 3, figsize=(13, 4), constrained_layout=True)
xs = [r["concurrency"] for r in values]
axes[0].plot(xs, [r["successful_rps"] for r in values], "o-", label="RPS")
tail = axes[0].twinx()
tail.plot(xs, [r["p95_ms"] for r in values], "s--", color="tab:orange", label="p95 ms")
tail.set_ylabel("p95 ms", color="tab:orange")
axes[0].set(title="Concurrency, baseline", xlabel="VUs", ylabel="Successful RPS")
db = sorted(
[optimized] + [r for r in rows if r["series"] == "database"],
key=lambda r: r["postgres_cpu"],
)
axes[1].plot([r["postgres_cpu"] for r in db], [r["successful_rps"] for r in db], "o-")
axes[1].set(
title="Controlled PostgreSQL saturation", xlabel="PostgreSQL CPU quota", ylabel="RPS"
)
axes[2].bar(
["Baseline", "Autocommit\nsingle SELECT"],
[baseline["successful_rps"], optimized["successful_rps"]],
color=["tab:blue", "tab:green"],
)
axes[2].set(title="Same 2 API / 2 CPU / 32 VUs", ylabel="Successful RPS")
gain = (optimized["successful_rps"] / baseline["successful_rps"] - 1) * 100
fig.suptitle(f"Bottlenecks and one change: median RPS gain {gain:+.1f}%")
fig.savefig(args.directory / "bottlenecks.png")
plt.close(fig)
print("\n".join(lines))
print(f"Optimization median gain: {gain:+.2f}%")
if __name__ == "__main__":
main()