Перенесите реализацию ЛР 4 на стенд ЛР 5
This commit is contained in:
Executable
+336
@@ -0,0 +1,336 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Tables and charts of track 1 from the raw results of run-series.sh.
|
||||
|
||||
python scripts/bench/plot.py <raw root> <output dir>
|
||||
|
||||
The raw root holds <series>/<config>/rep<k>/ directories. Written to the
|
||||
output directory:
|
||||
summary.csv one row per step of every run;
|
||||
capacity.csv the capacity of every run and the median per configuration;
|
||||
control.csv the noise checks of every run, the host speed probe and the
|
||||
loadgen CPU per request at the first step, each flagged when
|
||||
more than 25 % off its median;
|
||||
cost.csv CPU milliseconds per request of each component at the
|
||||
COST_RATES steps (valid, error-free, unflagged runs only), and
|
||||
the rate one database CPU would serve at that cost;
|
||||
<series>-throughput.png, <series>-p99.png, capacity.png,
|
||||
cpu-at-capacity.png, pool-waits.png, cost-per-request.png.
|
||||
Capacity is the highest step that passed the criterion loadgen applies (valid,
|
||||
under 1 % errors, p99 of successful requests within 500 ms). Charts show the
|
||||
median of the repeats; a hollow marker means the generator fell behind.
|
||||
"""
|
||||
import csv
|
||||
import gzip
|
||||
import json
|
||||
import pathlib
|
||||
import statistics
|
||||
import sys
|
||||
from datetime import datetime
|
||||
|
||||
import matplotlib
|
||||
|
||||
matplotlib.use("Agg")
|
||||
import matplotlib.pyplot as plt # noqa: E402
|
||||
|
||||
SLO_P99_MS = 500.0
|
||||
CONTROL_TOLERANCE = 0.25
|
||||
# Below saturation in every configuration, so the cost is not inflated by
|
||||
# queueing.
|
||||
COST_RATES = (800, 1000)
|
||||
COMPONENTS = ("api", "lb", "postgres", "valkey", "loadgen")
|
||||
|
||||
|
||||
def read_jsonl(path):
|
||||
packed = path.with_name(path.name + ".gz")
|
||||
if packed.exists():
|
||||
text = gzip.decompress(packed.read_bytes()).decode("utf-8")
|
||||
elif path.exists():
|
||||
text = path.read_text(encoding="utf-8")
|
||||
else:
|
||||
return []
|
||||
rows = []
|
||||
for line in text.splitlines():
|
||||
line = line.strip()
|
||||
if line:
|
||||
try:
|
||||
rows.append(json.loads(line))
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
return rows
|
||||
|
||||
|
||||
def component(name):
|
||||
for c in COMPONENTS:
|
||||
if f"-{c}-" in name:
|
||||
return c
|
||||
return None
|
||||
|
||||
|
||||
def parse_time(value):
|
||||
return datetime.fromisoformat(value.replace("Z", "+00:00")).timestamp()
|
||||
|
||||
|
||||
def load_runs(root):
|
||||
runs = []
|
||||
for steps_path in sorted(root.glob("*/*/rep*/steps.json")):
|
||||
d = steps_path.parent
|
||||
report = json.loads(steps_path.read_text(encoding="utf-8"))
|
||||
runs.append({
|
||||
"series": d.parent.parent.name,
|
||||
"config": d.parent.name,
|
||||
"rep": d.name,
|
||||
"start": parse_time(report["started_at"]),
|
||||
"steps": report["steps"],
|
||||
"docker": read_jsonl(d / "docker-stats.jsonl"),
|
||||
"metrics": read_jsonl(d / "metrics.jsonl"),
|
||||
"meta": json.loads((d / "meta.json").read_text(encoding="utf-8")) if (d / "meta.json").exists() else {},
|
||||
})
|
||||
return runs
|
||||
|
||||
|
||||
def mean_cpu(run, t0, t1):
|
||||
"""Mean CPU per component in [t0, t1); replicas of the API are summed."""
|
||||
per_time = {}
|
||||
for row in run["docker"]:
|
||||
c = component(row.get("name", ""))
|
||||
if c is None or not t0 <= row["t"] < t1:
|
||||
continue
|
||||
try:
|
||||
cpu = float(row["cpu"].rstrip("%"))
|
||||
except (ValueError, KeyError):
|
||||
continue
|
||||
per_time.setdefault(row["t"], {}).setdefault(c, 0.0)
|
||||
per_time[row["t"]][c] += cpu
|
||||
out = {}
|
||||
for c in COMPONENTS:
|
||||
values = [v[c] for v in per_time.values() if c in v]
|
||||
out[c] = round(statistics.mean(values), 1) if values else ""
|
||||
return out
|
||||
|
||||
|
||||
def pool_rate(run, t0, t1, counter):
|
||||
"""Growth of a pgxpool counter during [t0, t1), all replicas, per second."""
|
||||
first, last = {}, {}
|
||||
for row in run["metrics"]:
|
||||
t = run["start"] + row["t"]
|
||||
if not t0 <= t < t1:
|
||||
continue
|
||||
for replica in row.get("replicas") or []:
|
||||
pool = replica.get("pgxpool") or {}
|
||||
key = replica.get("instance")
|
||||
if key and counter in pool:
|
||||
first.setdefault(key, pool[counter])
|
||||
last[key] = pool[counter]
|
||||
if not last:
|
||||
return ""
|
||||
return round(sum(last[k] - first[k] for k in last) / max(t1 - t0, 1e-9), 1)
|
||||
|
||||
|
||||
def main(root, out):
|
||||
runs = load_runs(root)
|
||||
if not runs:
|
||||
sys.exit(f"no runs under {root}")
|
||||
out.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
rows, capacities = [], []
|
||||
for run in runs:
|
||||
best = None
|
||||
for step in run["steps"]:
|
||||
t0 = run["start"] + step["started_at_s"]
|
||||
t1 = t0 + step["seconds"]
|
||||
cpu = mean_cpu(run, t0, t1)
|
||||
classes = step["classes"]
|
||||
row = {
|
||||
"series": run["series"], "config": run["config"], "rep": run["rep"],
|
||||
"offered_rps": step["offered_rps"], "ok_rps": round(step["ok_rps"], 1),
|
||||
"error_share": round(step["error_share"], 4), "ok_p50_ms": step["ok_p50_ms"],
|
||||
"ok_p95_ms": step["ok_p95_ms"], "ok_p99_ms": step["ok_p99_ms"],
|
||||
"lag_p99_ms": step["lag_p99_ms"], "valid": step["valid"], "passed": step["passed"],
|
||||
"client_timeouts": classes.get("client_timeout", 0),
|
||||
"lb_errors": sum(n for c, n in classes.items() if c.endswith("_lb")),
|
||||
"api_errors": sum(n for c, n in classes.items() if c.startswith("http_") and not c.endswith("_lb")),
|
||||
**{f"cpu_{c}_pct": cpu[c] for c in COMPONENTS},
|
||||
"pool_waits_per_s": pool_rate(run, t0, t1, "empty_acquire_count"),
|
||||
"new_conns_per_s": pool_rate(run, t0, t1, "new_conns_count"),
|
||||
}
|
||||
rows.append(row)
|
||||
if step["passed"] and (best is None or step["offered_rps"] > best["offered_rps"]):
|
||||
best = row
|
||||
capacities.append({
|
||||
"series": run["series"], "config": run["config"], "rep": run["rep"],
|
||||
"capacity_rps": best["offered_rps"] if best else 0, "at": best,
|
||||
"invalid_below": sum(1 for s in run["steps"] if not s["valid"] and (best is None or s["offered_rps"] < best["offered_rps"])),
|
||||
})
|
||||
|
||||
with open(out / "summary.csv", "w", newline="", encoding="utf-8") as f:
|
||||
w = csv.DictWriter(f, fieldnames=list(rows[0]))
|
||||
w.writeheader()
|
||||
w.writerows(rows)
|
||||
|
||||
configs = []
|
||||
for c in capacities:
|
||||
if (c["series"], c["config"]) not in configs:
|
||||
configs.append((c["series"], c["config"]))
|
||||
with open(out / "capacity.csv", "w", newline="", encoding="utf-8") as f:
|
||||
w = csv.writer(f)
|
||||
w.writerow(["series", "config", "capacity_rps_per_repeat", "capacity_rps_median", "invalid_steps_below_capacity"])
|
||||
medians = {}
|
||||
for key in configs:
|
||||
mine = [c for c in capacities if (c["series"], c["config"]) == key]
|
||||
values = [c["capacity_rps"] for c in mine]
|
||||
medians[key] = statistics.median(values)
|
||||
w.writerow([key[0], key[1], " ".join(f"{v:g}" for v in values), f"{medians[key]:g}",
|
||||
sum(c["invalid_below"] for c in mine)])
|
||||
|
||||
control = []
|
||||
for run in runs:
|
||||
first = [r for r in rows if (r["series"], r["config"], r["rep"]) == (run["series"], run["config"], run["rep"])][:1]
|
||||
if first and first[0]["cpu_loadgen_pct"] != "":
|
||||
probe = run["meta"].get("host_probe_s", "")
|
||||
control.append([run["series"], run["config"], run["rep"], float(probe) if probe else None,
|
||||
round(float(first[0]["cpu_loadgen_pct"]) * 10000 / first[0]["offered_rps"])])
|
||||
if control:
|
||||
probes = [c[3] for c in control if c[3] is not None]
|
||||
probe_median = statistics.median(probes) if probes else None
|
||||
cpu_median = statistics.median(c[4] for c in control)
|
||||
with open(out / "control.csv", "w", newline="", encoding="utf-8") as f:
|
||||
w = csv.writer(f)
|
||||
w.writerow(["series", "config", "rep", "host_probe_s", "probe_deviation",
|
||||
"loadgen_cpu_ms_per_1000_requests", "cpu_deviation", "flagged"])
|
||||
flagged = set()
|
||||
for series, config, rep, probe, cpu in control:
|
||||
probe_dev = probe / probe_median - 1 if probe is not None else 0.0
|
||||
cpu_dev = cpu / cpu_median - 1
|
||||
noisy = max(abs(probe_dev), abs(cpu_dev)) > CONTROL_TOLERANCE
|
||||
if noisy:
|
||||
flagged.add((series, config, rep))
|
||||
w.writerow([series, config, rep, probe if probe is not None else "", f"{probe_dev:+.2f}", cpu,
|
||||
f"{cpu_dev:+.2f}", noisy])
|
||||
|
||||
cost = {}
|
||||
for r in rows:
|
||||
if (r["series"], r["config"], r["rep"]) in flagged or r["offered_rps"] not in COST_RATES:
|
||||
continue
|
||||
if not r["valid"] or r["error_share"] > 0:
|
||||
continue
|
||||
per = cost.setdefault((r["series"], r["config"]), {"runs": set()})
|
||||
per["runs"].add(r["rep"])
|
||||
for c in ("api", "postgres", "lb"):
|
||||
if r[f"cpu_{c}_pct"] != "":
|
||||
per.setdefault(c, []).append(float(r[f"cpu_{c}_pct"]) * 10 / r["offered_rps"])
|
||||
with open(out / "cost.csv", "w", newline="", encoding="utf-8") as f:
|
||||
w = csv.writer(f)
|
||||
w.writerow(["series", "config", "runs", "api_cpu_ms_per_request", "postgres_cpu_ms_per_request",
|
||||
"lb_cpu_ms_per_request", "rps_per_database_cpu"])
|
||||
for (series, config), per in cost.items():
|
||||
med = {c: statistics.median(per[c]) for c in ("api", "postgres", "lb") if per.get(c)}
|
||||
w.writerow([series, config, " ".join(sorted(per["runs"])),
|
||||
*(round(med[c], 2) if c in med else "" for c in ("api", "postgres", "lb")),
|
||||
round(1000 / med["postgres"]) if med.get("postgres") else ""])
|
||||
if cost:
|
||||
keys = list(cost)
|
||||
fig, ax = plt.subplots(figsize=(max(6, len(keys) * 0.9), 4.5))
|
||||
width = 0.27
|
||||
for i, c in enumerate(("api", "postgres", "lb")):
|
||||
ax.bar([j + i * width for j in range(len(keys))],
|
||||
[statistics.median(cost[k][c]) if cost[k].get(c) else 0 for k in keys], width, label=c)
|
||||
ax.set_xticks([j + width for j in range(len(keys))], [f"{s_}/{c_}" for s_, c_ in keys], rotation=30)
|
||||
ax.set_ylabel("CPU ms per request")
|
||||
ax.set_title(f"Cost per request at {'/'.join(map(str, COST_RATES))} rps (clean runs)")
|
||||
ax.grid(axis="y", alpha=0.3)
|
||||
ax.legend()
|
||||
fig.tight_layout()
|
||||
fig.savefig(out / "cost-per-request.png", dpi=120)
|
||||
plt.close(fig)
|
||||
|
||||
def median_by_offered(series, config, field):
|
||||
points, valid = {}, {}
|
||||
for r in rows:
|
||||
if r["series"] == series and r["config"] == config and r[field] != "":
|
||||
points.setdefault(r["offered_rps"], []).append(float(r[field]))
|
||||
valid[r["offered_rps"]] = valid.get(r["offered_rps"], True) and r["valid"]
|
||||
xs = sorted(points)
|
||||
return xs, [statistics.median(points[x]) for x in xs], [valid[x] for x in xs]
|
||||
|
||||
for series in sorted({s for s, _ in configs}):
|
||||
names = [c for s, c in configs if s == series]
|
||||
for field, title, filename, log in (
|
||||
("ok_rps", "Successful requests per second", "throughput", False),
|
||||
("ok_p99_ms", "p99 of successful requests, ms", "p99", True),
|
||||
):
|
||||
fig, ax = plt.subplots(figsize=(7, 4.5))
|
||||
top = 0
|
||||
for name in names:
|
||||
xs, ys, valid = median_by_offered(series, name, field)
|
||||
line, = ax.plot(xs, ys, label=name)
|
||||
ax.scatter([x for x, v in zip(xs, valid) if v], [y for y, v in zip(ys, valid) if v], color=line.get_color())
|
||||
ax.scatter([x for x, v in zip(xs, valid) if not v], [y for y, v in zip(ys, valid) if not v],
|
||||
facecolors="none", edgecolors=line.get_color())
|
||||
top = max([top, *xs])
|
||||
if field == "ok_rps" and top:
|
||||
ax.plot([0, top], [0, top], linestyle=":", color="grey", label="offered")
|
||||
if field == "ok_p99_ms":
|
||||
ax.axhline(SLO_P99_MS, linestyle="--", color="grey", label="500 ms")
|
||||
if log:
|
||||
ax.set_yscale("log")
|
||||
ax.set_xlabel("Offered requests per second")
|
||||
ax.set_ylabel(title)
|
||||
ax.set_title(f"Series {series}: median of repeats (hollow: generator fell behind)")
|
||||
ax.grid(alpha=0.3)
|
||||
ax.legend()
|
||||
fig.tight_layout()
|
||||
fig.savefig(out / f"{series}-{filename}.png", dpi=120)
|
||||
plt.close(fig)
|
||||
|
||||
labels = [f"{s}/{c}" for s, c in configs]
|
||||
fig, ax = plt.subplots(figsize=(max(6, len(labels) * 0.9), 4.5))
|
||||
meds = [medians[k] for k in configs]
|
||||
lows = [medians[k] - min(c["capacity_rps"] for c in capacities if (c["series"], c["config"]) == k) for k in configs]
|
||||
highs = [max(c["capacity_rps"] for c in capacities if (c["series"], c["config"]) == k) - medians[k] for k in configs]
|
||||
ax.bar(labels, meds, yerr=[lows, highs], capsize=4, color="#4c78a8")
|
||||
ax.set_ylabel("Capacity, requests per second")
|
||||
ax.set_title("Capacity per configuration (median, range of repeats)")
|
||||
ax.tick_params(axis="x", rotation=30)
|
||||
ax.grid(axis="y", alpha=0.3)
|
||||
fig.tight_layout()
|
||||
fig.savefig(out / "capacity.png", dpi=120)
|
||||
plt.close(fig)
|
||||
|
||||
fig, ax = plt.subplots(figsize=(max(6, len(labels) * 1.1), 4.5))
|
||||
width = 0.8 / len(COMPONENTS)
|
||||
for i, comp in enumerate(COMPONENTS):
|
||||
values = []
|
||||
for key in configs:
|
||||
at = [c["at"] for c in capacities if (c["series"], c["config"]) == key and c["at"]]
|
||||
cpu = [float(a[f"cpu_{comp}_pct"]) for a in at if a[f"cpu_{comp}_pct"] != ""]
|
||||
values.append(statistics.median(cpu) if cpu else 0)
|
||||
ax.bar([j + i * width for j in range(len(configs))], values, width, label=comp)
|
||||
ax.set_xticks([j + 0.4 - width / 2 for j in range(len(configs))], labels, rotation=30)
|
||||
ax.set_ylabel("Mean CPU at capacity, % of one core")
|
||||
ax.set_title("Which component is busy at capacity")
|
||||
ax.grid(axis="y", alpha=0.3)
|
||||
ax.legend()
|
||||
fig.tight_layout()
|
||||
fig.savefig(out / "cpu-at-capacity.png", dpi=120)
|
||||
plt.close(fig)
|
||||
|
||||
fig, ax = plt.subplots(figsize=(7, 4.5))
|
||||
for series, name in configs:
|
||||
xs, ys, _ = median_by_offered(series, name, "pool_waits_per_s")
|
||||
if xs:
|
||||
ax.plot(xs, ys, marker="o", label=f"{series}/{name}")
|
||||
ax.set_xlabel("Offered requests per second")
|
||||
ax.set_ylabel("Waits for a pooled connection per second")
|
||||
ax.set_title("Connection pool waits, all replicas")
|
||||
ax.grid(alpha=0.3)
|
||||
ax.legend()
|
||||
fig.tight_layout()
|
||||
fig.savefig(out / "pool-waits.png", dpi=120)
|
||||
plt.close(fig)
|
||||
print(f"{len(runs)} runs, {len(rows)} steps -> {out}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
if len(sys.argv) != 3:
|
||||
sys.exit(__doc__)
|
||||
main(pathlib.Path(sys.argv[1]), pathlib.Path(sys.argv[2]))
|
||||
Reference in New Issue
Block a user