Доработать инструменты экспериментов: чередовать повторы, останавливать лестницу на первой проваленной ступени, задавать maxconn и CPU базы в сериях, записывать замер скорости хоста и контроль шума, сократить комментарии

This commit is contained in:
2026-11-04 12:39:04 +03:00
parent ce47c138e4
commit baa5995938
24 changed files with 253 additions and 240 deletions
+48 -26
View File
@@ -5,14 +5,16 @@
The raw root holds <series>/<config>/rep<k>/ directories. Written to the
output directory:
summary.csv one row per step of every run: offered and successful rate,
errors, latency percentiles, generator lag, mean CPU of each
component and pool waits during the step;
summary.csv one row per step of every run;
capacity.csv the capacity of every run and the median per configuration;
control.csv the noise checks of every run: the host speed probe and the
loadgen CPU per request at the first step, flagged when more
than 25 % off the median;
<series>-throughput.png, <series>-p99.png, capacity.png,
cpu-at-capacity.png, pool-waits.png.
Capacity is the highest valid step (generator on schedule) with less than 1 %
errors and a p99 of successful requests within 500 ms.
Capacity is the highest step that passed the criterion loadgen applies (valid,
under 1 % errors, p99 of successful requests within 500 ms). Charts show the
median of the repeats; a hollow marker means the generator fell behind.
"""
import csv
import json
@@ -27,7 +29,7 @@ matplotlib.use("Agg")
import matplotlib.pyplot as plt # noqa: E402
SLO_P99_MS = 500.0
MAX_ERROR_SHARE = 0.01
CONTROL_TOLERANCE = 0.25
COMPONENTS = ("api", "lb", "postgres", "valkey", "loadgen")
@@ -69,6 +71,7 @@ def load_runs(root):
"steps": report["steps"],
"docker": read_jsonl(d / "docker-stats.jsonl"),
"metrics": read_jsonl(d / "metrics.jsonl"),
"meta": json.loads((d / "meta.json").read_text(encoding="utf-8")) if (d / "meta.json").exists() else {},
})
return runs
@@ -93,9 +96,8 @@ def mean_cpu(run, t0, t1):
return out
def pool_waits(run, t0, t1):
"""Requests that waited for a pooled connection during [t0, t1), all
replicas together, per second."""
def pool_rate(run, t0, t1, counter):
"""Growth of a pgxpool counter during [t0, t1), all replicas, per second."""
first, last = {}, {}
for row in run["metrics"]:
t = run["start"] + row["t"]
@@ -104,18 +106,14 @@ def pool_waits(run, t0, t1):
for replica in row.get("replicas") or []:
pool = replica.get("pgxpool") or {}
key = replica.get("instance")
if key and "empty_acquire_count" in pool:
first.setdefault(key, pool["empty_acquire_count"])
last[key] = pool["empty_acquire_count"]
if key and counter in pool:
first.setdefault(key, pool[counter])
last[key] = pool[counter]
if not last:
return ""
return round(sum(last[k] - first[k] for k in last) / max(t1 - t0, 1e-9), 1)
def is_capacity(step):
return step["valid"] and step["error_share"] < MAX_ERROR_SHARE and step["ok_p99_ms"] <= SLO_P99_MS
def main(root, out):
runs = load_runs(root)
if not runs:
@@ -129,17 +127,22 @@ def main(root, out):
t0 = run["start"] + step["started_at_s"]
t1 = t0 + step["seconds"]
cpu = mean_cpu(run, t0, t1)
classes = step["classes"]
row = {
"series": run["series"], "config": run["config"], "rep": run["rep"],
"offered_rps": step["offered_rps"], "ok_rps": round(step["ok_rps"], 1),
"error_share": round(step["error_share"], 4), "ok_p50_ms": step["ok_p50_ms"],
"ok_p95_ms": step["ok_p95_ms"], "ok_p99_ms": step["ok_p99_ms"],
"lag_p99_ms": step["lag_p99_ms"], "valid": step["valid"],
"lag_p99_ms": step["lag_p99_ms"], "valid": step["valid"], "passed": step["passed"],
"client_timeouts": classes.get("client_timeout", 0),
"lb_errors": sum(n for c, n in classes.items() if c.endswith("_lb")),
"api_errors": sum(n for c, n in classes.items() if c.startswith("http_") and not c.endswith("_lb")),
**{f"cpu_{c}_pct": cpu[c] for c in COMPONENTS},
"pool_waits_per_s": pool_waits(run, t0, t1),
"pool_waits_per_s": pool_rate(run, t0, t1, "empty_acquire_count"),
"new_conns_per_s": pool_rate(run, t0, t1, "new_conns_count"),
}
rows.append(row)
if is_capacity(step):
if step["passed"] and (best is None or step["offered_rps"] > best["offered_rps"]):
best = row
capacities.append({
"series": run["series"], "config": run["config"], "rep": run["rep"],
@@ -164,13 +167,29 @@ def main(root, out):
medians[key] = statistics.median(values)
w.writerow([key[0], key[1], " ".join(f"{v:g}" for v in values), f"{medians[key]:g}"])
control = []
for run in runs:
first = [r for r in rows if (r["series"], r["config"], r["rep"]) == (run["series"], run["config"], run["rep"])][:1]
if first and first[0]["cpu_loadgen_pct"] != "":
control.append([run["series"], run["config"], run["rep"], run["meta"].get("host_probe_s", ""), first[0]["offered_rps"],
round(float(first[0]["cpu_loadgen_pct"]) / first[0]["offered_rps"] * 1000, 1)])
if control:
median = statistics.median(c[5] for c in control)
with open(out / "control.csv", "w", newline="", encoding="utf-8") as f:
w = csv.writer(f)
w.writerow(["series", "config", "rep", "host_probe_s", "offered_rps", "loadgen_cpu_ms_per_1000_requests", "deviation", "flagged"])
for c in control:
deviation = c[5] / median - 1
w.writerow([*c, f"{deviation:+.2f}", abs(deviation) > CONTROL_TOLERANCE])
def median_by_offered(series, config, field):
points = {}
points, valid = {}, {}
for r in rows:
if r["series"] == series and r["config"] == config and r["valid"] and r[field] != "":
if r["series"] == series and r["config"] == config and r[field] != "":
points.setdefault(r["offered_rps"], []).append(float(r[field]))
valid[r["offered_rps"]] = valid.get(r["offered_rps"], True) and r["valid"]
xs = sorted(points)
return xs, [statistics.median(points[x]) for x in xs]
return xs, [statistics.median(points[x]) for x in xs], [valid[x] for x in xs]
for series in sorted({s for s, _ in configs}):
names = [c for s, c in configs if s == series]
@@ -181,8 +200,11 @@ def main(root, out):
fig, ax = plt.subplots(figsize=(7, 4.5))
top = 0
for name in names:
xs, ys = median_by_offered(series, name, field)
ax.plot(xs, ys, marker="o", label=name)
xs, ys, valid = median_by_offered(series, name, field)
line, = ax.plot(xs, ys, label=name)
ax.scatter([x for x, v in zip(xs, valid) if v], [y for y, v in zip(ys, valid) if v], color=line.get_color())
ax.scatter([x for x, v in zip(xs, valid) if not v], [y for y, v in zip(ys, valid) if not v],
facecolors="none", edgecolors=line.get_color())
top = max([top, *xs])
if field == "ok_rps" and top:
ax.plot([0, top], [0, top], linestyle=":", color="grey", label="offered")
@@ -192,7 +214,7 @@ def main(root, out):
ax.set_yscale("log")
ax.set_xlabel("Offered requests per second")
ax.set_ylabel(title)
ax.set_title(f"Series {series}: median of repeats, valid steps only")
ax.set_title(f"Series {series}: median of repeats (hollow: generator fell behind)")
ax.grid(alpha=0.3)
ax.legend()
fig.tight_layout()
@@ -233,7 +255,7 @@ def main(root, out):
fig, ax = plt.subplots(figsize=(7, 4.5))
for series, name in configs:
xs, ys = median_by_offered(series, name, "pool_waits_per_s")
xs, ys, _ = median_by_offered(series, name, "pool_waits_per_s")
if xs:
ax.plot(xs, ys, marker="o", label=f"{series}/{name}")
ax.set_xlabel("Offered requests per second")