Доработать инструменты экспериментов: чередовать повторы, останавливать лестницу на первой проваленной ступени, задавать maxconn и CPU базы в сериях, записывать замер скорости хоста и контроль шума, сократить комментарии
This commit is contained in:
+48
-26
@@ -5,14 +5,16 @@
|
||||
|
||||
The raw root holds <series>/<config>/rep<k>/ directories. Written to the
|
||||
output directory:
|
||||
summary.csv one row per step of every run: offered and successful rate,
|
||||
errors, latency percentiles, generator lag, mean CPU of each
|
||||
component and pool waits during the step;
|
||||
summary.csv one row per step of every run;
|
||||
capacity.csv the capacity of every run and the median per configuration;
|
||||
control.csv the noise checks of every run: the host speed probe and the
|
||||
loadgen CPU per request at the first step, flagged when more
|
||||
than 25 % off the median;
|
||||
<series>-throughput.png, <series>-p99.png, capacity.png,
|
||||
cpu-at-capacity.png, pool-waits.png.
|
||||
Capacity is the highest valid step (generator on schedule) with less than 1 %
|
||||
errors and a p99 of successful requests within 500 ms.
|
||||
Capacity is the highest step that passed the criterion loadgen applies (valid,
|
||||
under 1 % errors, p99 of successful requests within 500 ms). Charts show the
|
||||
median of the repeats; a hollow marker means the generator fell behind.
|
||||
"""
|
||||
import csv
|
||||
import json
|
||||
@@ -27,7 +29,7 @@ matplotlib.use("Agg")
|
||||
import matplotlib.pyplot as plt # noqa: E402
|
||||
|
||||
SLO_P99_MS = 500.0
|
||||
MAX_ERROR_SHARE = 0.01
|
||||
CONTROL_TOLERANCE = 0.25
|
||||
COMPONENTS = ("api", "lb", "postgres", "valkey", "loadgen")
|
||||
|
||||
|
||||
@@ -69,6 +71,7 @@ def load_runs(root):
|
||||
"steps": report["steps"],
|
||||
"docker": read_jsonl(d / "docker-stats.jsonl"),
|
||||
"metrics": read_jsonl(d / "metrics.jsonl"),
|
||||
"meta": json.loads((d / "meta.json").read_text(encoding="utf-8")) if (d / "meta.json").exists() else {},
|
||||
})
|
||||
return runs
|
||||
|
||||
@@ -93,9 +96,8 @@ def mean_cpu(run, t0, t1):
|
||||
return out
|
||||
|
||||
|
||||
def pool_waits(run, t0, t1):
|
||||
"""Requests that waited for a pooled connection during [t0, t1), all
|
||||
replicas together, per second."""
|
||||
def pool_rate(run, t0, t1, counter):
|
||||
"""Growth of a pgxpool counter during [t0, t1), all replicas, per second."""
|
||||
first, last = {}, {}
|
||||
for row in run["metrics"]:
|
||||
t = run["start"] + row["t"]
|
||||
@@ -104,18 +106,14 @@ def pool_waits(run, t0, t1):
|
||||
for replica in row.get("replicas") or []:
|
||||
pool = replica.get("pgxpool") or {}
|
||||
key = replica.get("instance")
|
||||
if key and "empty_acquire_count" in pool:
|
||||
first.setdefault(key, pool["empty_acquire_count"])
|
||||
last[key] = pool["empty_acquire_count"]
|
||||
if key and counter in pool:
|
||||
first.setdefault(key, pool[counter])
|
||||
last[key] = pool[counter]
|
||||
if not last:
|
||||
return ""
|
||||
return round(sum(last[k] - first[k] for k in last) / max(t1 - t0, 1e-9), 1)
|
||||
|
||||
|
||||
def is_capacity(step):
|
||||
return step["valid"] and step["error_share"] < MAX_ERROR_SHARE and step["ok_p99_ms"] <= SLO_P99_MS
|
||||
|
||||
|
||||
def main(root, out):
|
||||
runs = load_runs(root)
|
||||
if not runs:
|
||||
@@ -129,17 +127,22 @@ def main(root, out):
|
||||
t0 = run["start"] + step["started_at_s"]
|
||||
t1 = t0 + step["seconds"]
|
||||
cpu = mean_cpu(run, t0, t1)
|
||||
classes = step["classes"]
|
||||
row = {
|
||||
"series": run["series"], "config": run["config"], "rep": run["rep"],
|
||||
"offered_rps": step["offered_rps"], "ok_rps": round(step["ok_rps"], 1),
|
||||
"error_share": round(step["error_share"], 4), "ok_p50_ms": step["ok_p50_ms"],
|
||||
"ok_p95_ms": step["ok_p95_ms"], "ok_p99_ms": step["ok_p99_ms"],
|
||||
"lag_p99_ms": step["lag_p99_ms"], "valid": step["valid"],
|
||||
"lag_p99_ms": step["lag_p99_ms"], "valid": step["valid"], "passed": step["passed"],
|
||||
"client_timeouts": classes.get("client_timeout", 0),
|
||||
"lb_errors": sum(n for c, n in classes.items() if c.endswith("_lb")),
|
||||
"api_errors": sum(n for c, n in classes.items() if c.startswith("http_") and not c.endswith("_lb")),
|
||||
**{f"cpu_{c}_pct": cpu[c] for c in COMPONENTS},
|
||||
"pool_waits_per_s": pool_waits(run, t0, t1),
|
||||
"pool_waits_per_s": pool_rate(run, t0, t1, "empty_acquire_count"),
|
||||
"new_conns_per_s": pool_rate(run, t0, t1, "new_conns_count"),
|
||||
}
|
||||
rows.append(row)
|
||||
if is_capacity(step):
|
||||
if step["passed"] and (best is None or step["offered_rps"] > best["offered_rps"]):
|
||||
best = row
|
||||
capacities.append({
|
||||
"series": run["series"], "config": run["config"], "rep": run["rep"],
|
||||
@@ -164,13 +167,29 @@ def main(root, out):
|
||||
medians[key] = statistics.median(values)
|
||||
w.writerow([key[0], key[1], " ".join(f"{v:g}" for v in values), f"{medians[key]:g}"])
|
||||
|
||||
control = []
|
||||
for run in runs:
|
||||
first = [r for r in rows if (r["series"], r["config"], r["rep"]) == (run["series"], run["config"], run["rep"])][:1]
|
||||
if first and first[0]["cpu_loadgen_pct"] != "":
|
||||
control.append([run["series"], run["config"], run["rep"], run["meta"].get("host_probe_s", ""), first[0]["offered_rps"],
|
||||
round(float(first[0]["cpu_loadgen_pct"]) / first[0]["offered_rps"] * 1000, 1)])
|
||||
if control:
|
||||
median = statistics.median(c[5] for c in control)
|
||||
with open(out / "control.csv", "w", newline="", encoding="utf-8") as f:
|
||||
w = csv.writer(f)
|
||||
w.writerow(["series", "config", "rep", "host_probe_s", "offered_rps", "loadgen_cpu_ms_per_1000_requests", "deviation", "flagged"])
|
||||
for c in control:
|
||||
deviation = c[5] / median - 1
|
||||
w.writerow([*c, f"{deviation:+.2f}", abs(deviation) > CONTROL_TOLERANCE])
|
||||
|
||||
def median_by_offered(series, config, field):
|
||||
points = {}
|
||||
points, valid = {}, {}
|
||||
for r in rows:
|
||||
if r["series"] == series and r["config"] == config and r["valid"] and r[field] != "":
|
||||
if r["series"] == series and r["config"] == config and r[field] != "":
|
||||
points.setdefault(r["offered_rps"], []).append(float(r[field]))
|
||||
valid[r["offered_rps"]] = valid.get(r["offered_rps"], True) and r["valid"]
|
||||
xs = sorted(points)
|
||||
return xs, [statistics.median(points[x]) for x in xs]
|
||||
return xs, [statistics.median(points[x]) for x in xs], [valid[x] for x in xs]
|
||||
|
||||
for series in sorted({s for s, _ in configs}):
|
||||
names = [c for s, c in configs if s == series]
|
||||
@@ -181,8 +200,11 @@ def main(root, out):
|
||||
fig, ax = plt.subplots(figsize=(7, 4.5))
|
||||
top = 0
|
||||
for name in names:
|
||||
xs, ys = median_by_offered(series, name, field)
|
||||
ax.plot(xs, ys, marker="o", label=name)
|
||||
xs, ys, valid = median_by_offered(series, name, field)
|
||||
line, = ax.plot(xs, ys, label=name)
|
||||
ax.scatter([x for x, v in zip(xs, valid) if v], [y for y, v in zip(ys, valid) if v], color=line.get_color())
|
||||
ax.scatter([x for x, v in zip(xs, valid) if not v], [y for y, v in zip(ys, valid) if not v],
|
||||
facecolors="none", edgecolors=line.get_color())
|
||||
top = max([top, *xs])
|
||||
if field == "ok_rps" and top:
|
||||
ax.plot([0, top], [0, top], linestyle=":", color="grey", label="offered")
|
||||
@@ -192,7 +214,7 @@ def main(root, out):
|
||||
ax.set_yscale("log")
|
||||
ax.set_xlabel("Offered requests per second")
|
||||
ax.set_ylabel(title)
|
||||
ax.set_title(f"Series {series}: median of repeats, valid steps only")
|
||||
ax.set_title(f"Series {series}: median of repeats (hollow: generator fell behind)")
|
||||
ax.grid(alpha=0.3)
|
||||
ax.legend()
|
||||
fig.tight_layout()
|
||||
@@ -233,7 +255,7 @@ def main(root, out):
|
||||
|
||||
fig, ax = plt.subplots(figsize=(7, 4.5))
|
||||
for series, name in configs:
|
||||
xs, ys = median_by_offered(series, name, "pool_waits_per_s")
|
||||
xs, ys, _ = median_by_offered(series, name, "pool_waits_per_s")
|
||||
if xs:
|
||||
ax.plot(xs, ys, marker="o", label=f"{series}/{name}")
|
||||
ax.set_xlabel("Offered requests per second")
|
||||
|
||||
Reference in New Issue
Block a user