Files
mtusi2026lab3-fork/scripts/run-scaling-diagnostics.py
qrtx c1fec73ccc perf(bench): add scaling experiments and saturation diagnostics
Compare API and PostgreSQL CPU budgets and connection pool limits. Collect repeatable measurements and generate report aggregates and graphs.
2026-10-29 11:49:51 +03:00

326 lines
19 KiB
Python

#!/usr/bin/env python3
"""Run a bounded scaling sweep and collect diagnostics without HTTP metric endpoints."""
import argparse
from datetime import datetime, timezone
import json
from pathlib import Path
import subprocess
import time
import uuid
import hashlib
ROOT = Path(__file__).resolve().parents[1]
def command(args, timeout=180):
result = subprocess.run(args, cwd=ROOT, capture_output=True, text=True,
encoding="utf-8", errors="replace", timeout=timeout)
if result.returncode:
raise RuntimeError(f"{args[0]} {args[1]} failed: {result.stderr[-1500:]}")
return result.stdout
def compose(profile, diagnostics=True):
args = ["docker", "compose", "-f", "compose.yaml", "-f", f"configs/{profile}.yaml"]
if diagnostics:
args += ["-f", "test/compose.diagnostics.yaml"]
return args
def rows(path):
data = []
if path.exists():
text = path.read_text(encoding="utf-8")
decoder = json.JSONDecoder()
position = 0
while position < len(text):
if text[position].isspace():
position += 1
continue
try:
value, position = decoder.raw_decode(text, position)
data.append(value)
except json.JSONDecodeError:
break # Last record can be in progress when a running observer is copied.
return data
def delta(first, last, name):
return max(0, last.get(name, 0) - first.get(name, 0))
def resource_summary(samples):
if len(samples) < 2:
return {"samples": len(samples)}
first, last = samples[0], samples[-1]
elapsed = (last["time_ms"] - first["time_ms"]) / 1000
periods = delta(first, last, "cpu_nr_periods")
return {
"samples": len(samples), "coverage_seconds": elapsed,
"max_sample_gap_seconds": max((b["time_ms"] - a["time_ms"]) / 1000 for a, b in zip(samples, samples[1:])),
"mean_cpu_cores": delta(first, last, "cpu_usage_usec") / max(elapsed * 1e6, 1),
"throttled_seconds": delta(first, last, "cpu_throttled_usec") / 1e6,
"throttled_period_fraction": delta(first, last, "cpu_nr_throttled") / max(periods, 1),
"peak_memory_mib": max(s.get("memory_bytes", 0) or 0 for s in samples) / 1024**2,
"oom_kills": delta(first, last, "memory_oom_kill"),
}
def summarize(directory, benchmark):
start = benchmark["timing"]["measurement_started_epoch_ms"]
end = benchmark["timing"]["measurement_scheduled_end_epoch_ms"]
within = lambda data: [r for r in data if start <= r["time_ms"] <= end]
resources = {}
pools = {}
for path in sorted(directory.glob("api-*.jsonl")):
data = within(rows(path))
flattened = [{**r, **{f"cpu_{k}": v for k, v in (r.get("cpu") or {}).items()},
**{f"memory_{k}": v for k, v in (r.get("memory_events") or {}).items()}}
for r in data]
resources[path.stem] = resource_summary(flattened)
if len(data) < 2:
continue
first, last = data[0]["pools"], data[-1]["pools"]
acquisitions = sum(p["acquisitions"] for p in last) - sum(p["acquisitions"] for p in first)
acquire_ms = sum(p["acquire_ms"] for p in last) - sum(p["acquire_ms"] for p in first)
intervals = [p for r in data for p in r["pools"]]
totals = [sum(p["total"] for p in r["pools"]) for r in data]
pools[path.stem] = {
"samples": len(data), "acquisitions": acquisitions,
"acquisition_errors": sum(p["errors"] for p in last) - sum(p["errors"] for p in first),
"mean_acquisition_ms": acquire_ms / max(acquisitions, 1),
"max_acquisition_ms": max((p["interval_max_ms"] for p in intervals), default=0),
"max_waiting": max((p["interval_max_waiting"] for p in intervals), default=0),
"queued_interval_fraction": sum(p["interval_max_waiting"] > 0 for p in intervals) / max(len(intervals), 1),
"max_total": max((p["total"] for p in intervals), default=0),
"observed_connection_increases": sum(max(0, b-a) for a, b in zip(totals, totals[1:])),
"observed_connection_decreases": sum(max(0, a-b) for a, b in zip(totals, totals[1:])),
"event_loop_p99_max_ms": max(r["event_loop_delay_ms"]["p99"] for r in data),
}
pg = within(rows(directory / "postgres.jsonl"))
pg_samples = [r for r in pg if r["kind"] == "postgres"]
pg_samples += within(rows(directory / "postgres-sql.jsonl"))
pg_samples.sort(key=lambda r: r["time_ms"])
host_samples = [r for r in pg if r["kind"] == "host"]
host = {"samples": len(host_samples)}
if len(host_samples) > 1:
host_seconds = (host_samples[-1]["time_ms"] - host_samples[0]["time_ms"])/1000
host.update({key + "_fraction": delta(host_samples[0], host_samples[-1], key + "_total_usec") / max(host_seconds*1e6, 1)
for key in ("cpu_some", "memory_some", "memory_full", "io_some", "io_full")})
host["min_available_mib"] = min(r["MemAvailable_kib"] for r in host_samples)/1024
host["max_swap_used_mib"] = max(r["SwapTotal_kib"] - r["SwapFree_kib"] for r in host_samples)/1024
waits = {}
max_active = max_idle_tx = 0
for sample in pg_samples:
max_active = max(max_active, sum(a["count"] for a in sample["activity"] if a["state"] == "active"))
max_idle_tx = max(max_idle_tx, sum(a["count"] for a in sample["activity"] if a["state"] == "idle in transaction"))
for a in sample["activity"]:
if a["state"] == "active":
key = f"{a['wait_event_type'] or 'no_wait_event'}/{a['wait_event'] or '-'}"
waits[key] = waits.get(key, 0) + a["count"]
for role in ("postgres", "nginx", "redis"):
resources[role] = resource_summary([r for r in within(rows(directory / f"{role}.jsonl")) if r["kind"] == "cgroup"])
upstream = [r for r in within(rows(directory / "nginx.jsonl")) if r["kind"] == "upstream"]
changes = sum(a["servers"] != b["servers"] for a, b in zip(upstream, upstream[1:]))
m = benchmark["measurement"]
return {
"label": benchmark["label"], "profile": benchmark["profile"],
"measurement": m, "client": benchmark["client"],
"resources": resources, "pools": pools,
"host": host,
"postgres": {"samples": len(pg_samples), "max_active": max_active,
"max_idle_in_transaction": max_idle_tx, "active_wait_observations": waits},
"nginx": {"samples": len(upstream), "upstream_changes": changes,
"min_ready": min((len(r["servers"]) for r in upstream), default=None),
"max_ready": max((len(r["servers"]) for r in upstream), default=None)},
"limits_reached": {
"generator_late": m["scheduler_late_slots"] / m["scheduled_slots"] > 0.05,
"client_capacity": (m["client_dropped_slots"] - m["scheduler_late_slots"]) / m["scheduled_slots"] > 0.05,
"request_errors": m["all"]["errors"] / max(m["all"]["attempted"], 1) > 0.01,
"latency": (m["all"]["successful_latency_ms"]["p95"] or 0) > 1000,
},
}
def run_one(config, profile, rate, repeat, containers):
label = f"{profile}-r{rate}-repeat{repeat}"
directory = config.output / label
directory.mkdir(parents=True, exist_ok=False)
workers = []
stop = f"/tmp/scaling-observer-{uuid.uuid4().hex}.stop"
observer = (ROOT / "scripts/diagnostics/observer.sh").read_bytes()
client_container = None
try:
for role in ("postgres", "nginx", "redis"):
output = (directory / f"{role}.jsonl").open("wb")
errors = (directory / f"{role}.stderr").open("wb")
child = subprocess.Popen(["docker", "exec", "-i", containers[role][0],
"sh", "-s", role, stop], cwd=ROOT,
stdin=subprocess.PIPE, stdout=output, stderr=errors)
workers.append((role, child, output, errors))
child.stdin.write(observer)
child.stdin.close()
started = datetime.now(timezone.utc).isoformat()
target = directory / "benchmark.json"
args = ["node", "scripts/benchmark-scaling.mjs", "--base-url", config.base_url,
"--label", label, "--rate", str(rate), "--duration-seconds", str(config.duration_seconds),
"--warmup-seconds", str(config.warmup_seconds), "--seed-messages", "30",
"--expected-instances", str({"one": 1, "two": 2, "four": 4}[profile]),
"--output", str(target)]
if config.client == "container":
nginx = json.loads(command(["docker", "inspect", containers["nginx"][0]]))[0]
api = json.loads(command(["docker", "inspect", containers["api"][0]]))[0]
networks = set(nginx["NetworkSettings"]["Networks"]) & set(api["NetworkSettings"]["Networks"])
if len(networks) != 1:
raise RuntimeError("expected one common API/nginx network")
client_container = "lab3-diagnostic-client-" + uuid.uuid4().hex
args[args.index("--base-url") + 1] = config.container_url
args[args.index("--output") + 1] = "/tmp/scaling-benchmark.json"
args[1] = "/benchmark.mjs"
args = ["docker", "run", "--name", client_container, "--no-healthcheck",
"--network", next(iter(networks)),
"-v", f"{ROOT / 'scripts/benchmark-scaling.mjs'}:/benchmark.mjs:ro",
"--entrypoint", "node", api["Image"], *args[1:]]
result = subprocess.run(args, cwd=ROOT, capture_output=True, text=True,
encoding="utf-8", errors="replace",
timeout=config.duration_seconds + config.warmup_seconds + 120)
(directory / "benchmark.stderr").write_text(result.stderr, encoding="utf-8")
if client_container is not None:
command(["docker", "cp", f"{client_container}:/tmp/scaling-benchmark.json", str(target)])
if not target.exists():
raise RuntimeError(f"{label}: benchmark did not produce results: {result.stderr}")
benchmark = json.loads(target.read_text(encoding="utf-8"))
for index, container in enumerate(containers["api"], 1):
command(["docker", "cp", f"{container}:/tmp/scaling-pool.jsonl", str(directory / f"api-{index}.jsonl")])
ended = datetime.now(timezone.utc).isoformat()
# Docker puts nginx stderr notices on CLI stderr; collect both streams.
log_result = subprocess.run(["docker", "logs", "--timestamps", "--since", started,
"--until", ended, containers["nginx"][0]], cwd=ROOT,
capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=30)
diagnostic_lines = [line for line in (log_result.stdout + log_result.stderr).splitlines()
if any(key in line for key in ("signal process started", "download timed out",
"no live upstreams", "upstream timed out"))]
(directory / "nginx-events.log").write_text("\n".join(diagnostic_lines), encoding="utf-8")
(directory / "execution.json").write_text(json.dumps({"returncode": result.returncode,
"started": started, "ended": ended, "command": args}, indent=2), encoding="utf-8")
finally:
if client_container is not None:
# Only the uniquely named disposable generator created by this run.
subprocess.run(["docker", "rm", "-f", client_container], cwd=ROOT, capture_output=True, timeout=30)
for role, child, output, errors in workers:
try:
command(["docker", "exec", containers[role][0], "touch", stop], timeout=15)
child.wait(timeout=15)
except (RuntimeError, subprocess.TimeoutExpired):
child.terminate()
child.wait(timeout=10)
output.close()
errors.close()
if role == "postgres":
command(["docker", "cp", f"{containers[role][0]}:{stop}.pg.jsonl", str(directory / "postgres-sql.jsonl")])
command(["docker", "exec", containers[role][0], "rm", "-f", stop + ".pg.jsonl"])
summary = summarize(directory, benchmark)
summary["diagnostics_complete"] = (
len(summary["pools"]) == len(containers["api"])
and summary["postgres"]["samples"] > 0
and summary["nginx"]["samples"] > 0
and all(v["samples"] > 0 for v in summary["resources"].values())
)
(directory / "summary.json").write_text(json.dumps(summary, indent=2) + "\n", encoding="utf-8")
if not summary["diagnostics_complete"]:
raise RuntimeError(f"{label}: diagnostics incomplete; inspect observer stderr files")
m = summary["measurement"]
print(f"DONE {label}: RPS={m['all']['successful_rps']} p95={m['all']['successful_latency_ms']['p95']}ms "
f"errors={m['all']['errors']} drops={m['client_dropped_slots']}/{m['scheduled_slots']} "
f"pool_wait_max={max((p['max_waiting'] for p in summary['pools'].values()), default=0)}", flush=True)
return summary
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--base-url", default="http://127.0.0.1:18080")
parser.add_argument("--client", choices=("host", "container"), default="host")
parser.add_argument("--container-url", default="http://nginx")
parser.add_argument("--profiles", nargs="+", choices=("one", "two", "four"), default=["one", "two", "four"])
parser.add_argument("--rates", nargs="+", type=int, default=[100], help="Scheduled requests/s (default: 100 only)")
parser.add_argument("--duration-seconds", type=int, default=60, help="Measured phase per run (default: 60 seconds)")
parser.add_argument("--warmup-seconds", type=int, default=15, help="Read-only warmup per run (default: 15 seconds)")
parser.add_argument("--repetitions", type=int, default=3)
parser.add_argument("--full-matrix", action="store_true", help="Run all requested rates; record threshold crossings instead of stopping a profile")
parser.add_argument("--output", type=Path, required=True)
config = parser.parse_args()
if config.repetitions < 1 or config.duration_seconds < 1 or config.warmup_seconds < 0 or any(r < 1 or r > 1000 for r in config.rates):
parser.error("invalid duration, repetitions or rates")
config.output = config.output.resolve()
config.output.mkdir(parents=True, exist_ok=False)
manifest = {"started": datetime.now(timezone.utc).isoformat(), "profiles": config.profiles,
"client": config.client,
"rates": config.rates, "duration_seconds": config.duration_seconds,
"warmup_seconds": config.warmup_seconds, "repetitions": config.repetitions,
"full_matrix": config.full_matrix,
"stop_rule": ("record only; continue all requested rates after >=2 repeats exceed thresholds: " if config.full_matrix
else "stop a profile after >=2 repeats at one rate exceed any threshold: ") +
"scheduler late >5%, capacity drops >5%, request errors >1%, success p95 >1000ms",
"sampling": "API queue every 100ms; API JSON/cgroup about 1s; other services and PG about 2s",
"observer_version": 3,
"source_sha256": {name: hashlib.sha256((ROOT/name).read_bytes()).hexdigest() for name in (
"scripts/diagnostics/pool.cjs", "scripts/diagnostics/observer.sh",
"scripts/benchmark-scaling.mjs", "scripts/run-scaling-diagnostics.py")},
"notes": ["All profiles use the same instrumentation; actual per-replica pool limits are recorded in containers.",
"A container client uses the nginx service directly; it shares Docker VM CPU and bypasses Windows port forwarding.",
"Acquisition latency includes opening new connections; it is not SQL execution time.",
"Observers consume resources inside the existing container budgets.",
"Instrumentation stores no tokens, passwords, message text or SQL parameters."],
"containers": {}, "environments": {}, "runs": [], "stopped_profiles": {}, "threshold_crossings": {}}
manifest_path = config.output / "manifest.json"
save = lambda: manifest_path.write_text(json.dumps(manifest, indent=2) + "\n", encoding="utf-8")
save()
try:
for profile in config.profiles:
print(f"SWITCH {profile}", flush=True)
command(compose(profile) + ["up", "-d", "--no-build", "--no-deps", "--wait", "--wait-timeout", "120", "api"])
time.sleep(6) # Allow readiness's two successes and nginx reload to settle.
containers = {}
for role in ("api", "postgres", "nginx", "redis"):
containers[role] = command(compose(profile) + ["ps", "-q", role]).split()
if not containers[role]:
raise RuntimeError(f"no running {role}")
details = json.loads(command(["docker", "inspect", *[c for ids in containers.values() for c in ids]]))
running = [json.loads(line) for line in command(["docker", "ps", "--format", "{{json .}}"]).splitlines()]
manifest["environments"][profile] = {
"docker": json.loads(command(["docker", "info", "--format",
'{"cpus":{{.NCPU}},"memory_bytes":{{.MemTotal}},"version":{{json .ServerVersion}},"kernel":{{json .KernelVersion}}}'])),
"running_containers": [{"name": row.get("Names"), "image": row.get("Image")} for row in running],
}
manifest["containers"][profile] = [{"id": c["Id"], "name": c["Name"], "image": c["Image"],
"nano_cpus": c["HostConfig"]["NanoCpus"], "memory_bytes": c["HostConfig"]["Memory"],
"pool_max": next((e.split("=", 1)[1] for e in c["Config"]["Env"] if e.startswith("DB_POOL_MAX=")), None)} for c in details]
save()
for rate in config.rates:
batch = []
for repeat in range(1, config.repetitions + 1):
print(f"START {profile} rate={rate} repeat={repeat}", flush=True)
summary = run_one(config, profile, rate, repeat, containers)
batch.append(summary)
manifest["runs"].append(summary["label"])
save()
reasons = [key for key in batch[0]["limits_reached"]
if sum(r["limits_reached"][key] for r in batch) >= min(2, config.repetitions)]
if reasons:
manifest["threshold_crossings"].setdefault(profile, []).append({"rate": rate, "reasons": reasons})
if not config.full_matrix:
manifest["stopped_profiles"][profile] = {"rate": rate, "reasons": reasons}
print(f"{'CONTINUE (full matrix)' if config.full_matrix else 'STOP'} {profile} at {rate}: {', '.join(reasons)}", flush=True)
save()
if not config.full_matrix:
break
finally:
print("RESTORE two (without diagnostic preload)", flush=True)
command(compose("two", diagnostics=False) + ["up", "-d", "--no-build", "--no-deps", "--wait", "--wait-timeout", "120", "api"])
manifest["finished"] = datetime.now(timezone.utc).isoformat()
save()
if __name__ == "__main__":
main()