forked from mtusi2026/mtusi2026lab3
Compare API and PostgreSQL CPU budgets and connection pool limits. Collect repeatable measurements and generate report aggregates and graphs.
326 lines
19 KiB
Python
326 lines
19 KiB
Python
#!/usr/bin/env python3
|
|
"""Run a bounded scaling sweep and collect diagnostics without HTTP metric endpoints."""
|
|
import argparse
|
|
from datetime import datetime, timezone
|
|
import json
|
|
from pathlib import Path
|
|
import subprocess
|
|
import time
|
|
import uuid
|
|
import hashlib
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
|
|
|
|
def command(args, timeout=180):
|
|
result = subprocess.run(args, cwd=ROOT, capture_output=True, text=True,
|
|
encoding="utf-8", errors="replace", timeout=timeout)
|
|
if result.returncode:
|
|
raise RuntimeError(f"{args[0]} {args[1]} failed: {result.stderr[-1500:]}")
|
|
return result.stdout
|
|
|
|
|
|
def compose(profile, diagnostics=True):
|
|
args = ["docker", "compose", "-f", "compose.yaml", "-f", f"configs/{profile}.yaml"]
|
|
if diagnostics:
|
|
args += ["-f", "test/compose.diagnostics.yaml"]
|
|
return args
|
|
|
|
|
|
def rows(path):
|
|
data = []
|
|
if path.exists():
|
|
text = path.read_text(encoding="utf-8")
|
|
decoder = json.JSONDecoder()
|
|
position = 0
|
|
while position < len(text):
|
|
if text[position].isspace():
|
|
position += 1
|
|
continue
|
|
try:
|
|
value, position = decoder.raw_decode(text, position)
|
|
data.append(value)
|
|
except json.JSONDecodeError:
|
|
break # Last record can be in progress when a running observer is copied.
|
|
return data
|
|
|
|
|
|
def delta(first, last, name):
|
|
return max(0, last.get(name, 0) - first.get(name, 0))
|
|
|
|
|
|
def resource_summary(samples):
|
|
if len(samples) < 2:
|
|
return {"samples": len(samples)}
|
|
first, last = samples[0], samples[-1]
|
|
elapsed = (last["time_ms"] - first["time_ms"]) / 1000
|
|
periods = delta(first, last, "cpu_nr_periods")
|
|
return {
|
|
"samples": len(samples), "coverage_seconds": elapsed,
|
|
"max_sample_gap_seconds": max((b["time_ms"] - a["time_ms"]) / 1000 for a, b in zip(samples, samples[1:])),
|
|
"mean_cpu_cores": delta(first, last, "cpu_usage_usec") / max(elapsed * 1e6, 1),
|
|
"throttled_seconds": delta(first, last, "cpu_throttled_usec") / 1e6,
|
|
"throttled_period_fraction": delta(first, last, "cpu_nr_throttled") / max(periods, 1),
|
|
"peak_memory_mib": max(s.get("memory_bytes", 0) or 0 for s in samples) / 1024**2,
|
|
"oom_kills": delta(first, last, "memory_oom_kill"),
|
|
}
|
|
|
|
|
|
def summarize(directory, benchmark):
|
|
start = benchmark["timing"]["measurement_started_epoch_ms"]
|
|
end = benchmark["timing"]["measurement_scheduled_end_epoch_ms"]
|
|
within = lambda data: [r for r in data if start <= r["time_ms"] <= end]
|
|
resources = {}
|
|
pools = {}
|
|
for path in sorted(directory.glob("api-*.jsonl")):
|
|
data = within(rows(path))
|
|
flattened = [{**r, **{f"cpu_{k}": v for k, v in (r.get("cpu") or {}).items()},
|
|
**{f"memory_{k}": v for k, v in (r.get("memory_events") or {}).items()}}
|
|
for r in data]
|
|
resources[path.stem] = resource_summary(flattened)
|
|
if len(data) < 2:
|
|
continue
|
|
first, last = data[0]["pools"], data[-1]["pools"]
|
|
acquisitions = sum(p["acquisitions"] for p in last) - sum(p["acquisitions"] for p in first)
|
|
acquire_ms = sum(p["acquire_ms"] for p in last) - sum(p["acquire_ms"] for p in first)
|
|
intervals = [p for r in data for p in r["pools"]]
|
|
totals = [sum(p["total"] for p in r["pools"]) for r in data]
|
|
pools[path.stem] = {
|
|
"samples": len(data), "acquisitions": acquisitions,
|
|
"acquisition_errors": sum(p["errors"] for p in last) - sum(p["errors"] for p in first),
|
|
"mean_acquisition_ms": acquire_ms / max(acquisitions, 1),
|
|
"max_acquisition_ms": max((p["interval_max_ms"] for p in intervals), default=0),
|
|
"max_waiting": max((p["interval_max_waiting"] for p in intervals), default=0),
|
|
"queued_interval_fraction": sum(p["interval_max_waiting"] > 0 for p in intervals) / max(len(intervals), 1),
|
|
"max_total": max((p["total"] for p in intervals), default=0),
|
|
"observed_connection_increases": sum(max(0, b-a) for a, b in zip(totals, totals[1:])),
|
|
"observed_connection_decreases": sum(max(0, a-b) for a, b in zip(totals, totals[1:])),
|
|
"event_loop_p99_max_ms": max(r["event_loop_delay_ms"]["p99"] for r in data),
|
|
}
|
|
pg = within(rows(directory / "postgres.jsonl"))
|
|
pg_samples = [r for r in pg if r["kind"] == "postgres"]
|
|
pg_samples += within(rows(directory / "postgres-sql.jsonl"))
|
|
pg_samples.sort(key=lambda r: r["time_ms"])
|
|
host_samples = [r for r in pg if r["kind"] == "host"]
|
|
host = {"samples": len(host_samples)}
|
|
if len(host_samples) > 1:
|
|
host_seconds = (host_samples[-1]["time_ms"] - host_samples[0]["time_ms"])/1000
|
|
host.update({key + "_fraction": delta(host_samples[0], host_samples[-1], key + "_total_usec") / max(host_seconds*1e6, 1)
|
|
for key in ("cpu_some", "memory_some", "memory_full", "io_some", "io_full")})
|
|
host["min_available_mib"] = min(r["MemAvailable_kib"] for r in host_samples)/1024
|
|
host["max_swap_used_mib"] = max(r["SwapTotal_kib"] - r["SwapFree_kib"] for r in host_samples)/1024
|
|
waits = {}
|
|
max_active = max_idle_tx = 0
|
|
for sample in pg_samples:
|
|
max_active = max(max_active, sum(a["count"] for a in sample["activity"] if a["state"] == "active"))
|
|
max_idle_tx = max(max_idle_tx, sum(a["count"] for a in sample["activity"] if a["state"] == "idle in transaction"))
|
|
for a in sample["activity"]:
|
|
if a["state"] == "active":
|
|
key = f"{a['wait_event_type'] or 'no_wait_event'}/{a['wait_event'] or '-'}"
|
|
waits[key] = waits.get(key, 0) + a["count"]
|
|
for role in ("postgres", "nginx", "redis"):
|
|
resources[role] = resource_summary([r for r in within(rows(directory / f"{role}.jsonl")) if r["kind"] == "cgroup"])
|
|
upstream = [r for r in within(rows(directory / "nginx.jsonl")) if r["kind"] == "upstream"]
|
|
changes = sum(a["servers"] != b["servers"] for a, b in zip(upstream, upstream[1:]))
|
|
m = benchmark["measurement"]
|
|
return {
|
|
"label": benchmark["label"], "profile": benchmark["profile"],
|
|
"measurement": m, "client": benchmark["client"],
|
|
"resources": resources, "pools": pools,
|
|
"host": host,
|
|
"postgres": {"samples": len(pg_samples), "max_active": max_active,
|
|
"max_idle_in_transaction": max_idle_tx, "active_wait_observations": waits},
|
|
"nginx": {"samples": len(upstream), "upstream_changes": changes,
|
|
"min_ready": min((len(r["servers"]) for r in upstream), default=None),
|
|
"max_ready": max((len(r["servers"]) for r in upstream), default=None)},
|
|
"limits_reached": {
|
|
"generator_late": m["scheduler_late_slots"] / m["scheduled_slots"] > 0.05,
|
|
"client_capacity": (m["client_dropped_slots"] - m["scheduler_late_slots"]) / m["scheduled_slots"] > 0.05,
|
|
"request_errors": m["all"]["errors"] / max(m["all"]["attempted"], 1) > 0.01,
|
|
"latency": (m["all"]["successful_latency_ms"]["p95"] or 0) > 1000,
|
|
},
|
|
}
|
|
|
|
|
|
def run_one(config, profile, rate, repeat, containers):
|
|
label = f"{profile}-r{rate}-repeat{repeat}"
|
|
directory = config.output / label
|
|
directory.mkdir(parents=True, exist_ok=False)
|
|
workers = []
|
|
stop = f"/tmp/scaling-observer-{uuid.uuid4().hex}.stop"
|
|
observer = (ROOT / "scripts/diagnostics/observer.sh").read_bytes()
|
|
client_container = None
|
|
try:
|
|
for role in ("postgres", "nginx", "redis"):
|
|
output = (directory / f"{role}.jsonl").open("wb")
|
|
errors = (directory / f"{role}.stderr").open("wb")
|
|
child = subprocess.Popen(["docker", "exec", "-i", containers[role][0],
|
|
"sh", "-s", role, stop], cwd=ROOT,
|
|
stdin=subprocess.PIPE, stdout=output, stderr=errors)
|
|
workers.append((role, child, output, errors))
|
|
child.stdin.write(observer)
|
|
child.stdin.close()
|
|
started = datetime.now(timezone.utc).isoformat()
|
|
target = directory / "benchmark.json"
|
|
args = ["node", "scripts/benchmark-scaling.mjs", "--base-url", config.base_url,
|
|
"--label", label, "--rate", str(rate), "--duration-seconds", str(config.duration_seconds),
|
|
"--warmup-seconds", str(config.warmup_seconds), "--seed-messages", "30",
|
|
"--expected-instances", str({"one": 1, "two": 2, "four": 4}[profile]),
|
|
"--output", str(target)]
|
|
if config.client == "container":
|
|
nginx = json.loads(command(["docker", "inspect", containers["nginx"][0]]))[0]
|
|
api = json.loads(command(["docker", "inspect", containers["api"][0]]))[0]
|
|
networks = set(nginx["NetworkSettings"]["Networks"]) & set(api["NetworkSettings"]["Networks"])
|
|
if len(networks) != 1:
|
|
raise RuntimeError("expected one common API/nginx network")
|
|
client_container = "lab3-diagnostic-client-" + uuid.uuid4().hex
|
|
args[args.index("--base-url") + 1] = config.container_url
|
|
args[args.index("--output") + 1] = "/tmp/scaling-benchmark.json"
|
|
args[1] = "/benchmark.mjs"
|
|
args = ["docker", "run", "--name", client_container, "--no-healthcheck",
|
|
"--network", next(iter(networks)),
|
|
"-v", f"{ROOT / 'scripts/benchmark-scaling.mjs'}:/benchmark.mjs:ro",
|
|
"--entrypoint", "node", api["Image"], *args[1:]]
|
|
result = subprocess.run(args, cwd=ROOT, capture_output=True, text=True,
|
|
encoding="utf-8", errors="replace",
|
|
timeout=config.duration_seconds + config.warmup_seconds + 120)
|
|
(directory / "benchmark.stderr").write_text(result.stderr, encoding="utf-8")
|
|
if client_container is not None:
|
|
command(["docker", "cp", f"{client_container}:/tmp/scaling-benchmark.json", str(target)])
|
|
if not target.exists():
|
|
raise RuntimeError(f"{label}: benchmark did not produce results: {result.stderr}")
|
|
benchmark = json.loads(target.read_text(encoding="utf-8"))
|
|
for index, container in enumerate(containers["api"], 1):
|
|
command(["docker", "cp", f"{container}:/tmp/scaling-pool.jsonl", str(directory / f"api-{index}.jsonl")])
|
|
ended = datetime.now(timezone.utc).isoformat()
|
|
# Docker puts nginx stderr notices on CLI stderr; collect both streams.
|
|
log_result = subprocess.run(["docker", "logs", "--timestamps", "--since", started,
|
|
"--until", ended, containers["nginx"][0]], cwd=ROOT,
|
|
capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=30)
|
|
diagnostic_lines = [line for line in (log_result.stdout + log_result.stderr).splitlines()
|
|
if any(key in line for key in ("signal process started", "download timed out",
|
|
"no live upstreams", "upstream timed out"))]
|
|
(directory / "nginx-events.log").write_text("\n".join(diagnostic_lines), encoding="utf-8")
|
|
(directory / "execution.json").write_text(json.dumps({"returncode": result.returncode,
|
|
"started": started, "ended": ended, "command": args}, indent=2), encoding="utf-8")
|
|
finally:
|
|
if client_container is not None:
|
|
# Only the uniquely named disposable generator created by this run.
|
|
subprocess.run(["docker", "rm", "-f", client_container], cwd=ROOT, capture_output=True, timeout=30)
|
|
for role, child, output, errors in workers:
|
|
try:
|
|
command(["docker", "exec", containers[role][0], "touch", stop], timeout=15)
|
|
child.wait(timeout=15)
|
|
except (RuntimeError, subprocess.TimeoutExpired):
|
|
child.terminate()
|
|
child.wait(timeout=10)
|
|
output.close()
|
|
errors.close()
|
|
if role == "postgres":
|
|
command(["docker", "cp", f"{containers[role][0]}:{stop}.pg.jsonl", str(directory / "postgres-sql.jsonl")])
|
|
command(["docker", "exec", containers[role][0], "rm", "-f", stop + ".pg.jsonl"])
|
|
summary = summarize(directory, benchmark)
|
|
summary["diagnostics_complete"] = (
|
|
len(summary["pools"]) == len(containers["api"])
|
|
and summary["postgres"]["samples"] > 0
|
|
and summary["nginx"]["samples"] > 0
|
|
and all(v["samples"] > 0 for v in summary["resources"].values())
|
|
)
|
|
(directory / "summary.json").write_text(json.dumps(summary, indent=2) + "\n", encoding="utf-8")
|
|
if not summary["diagnostics_complete"]:
|
|
raise RuntimeError(f"{label}: diagnostics incomplete; inspect observer stderr files")
|
|
m = summary["measurement"]
|
|
print(f"DONE {label}: RPS={m['all']['successful_rps']} p95={m['all']['successful_latency_ms']['p95']}ms "
|
|
f"errors={m['all']['errors']} drops={m['client_dropped_slots']}/{m['scheduled_slots']} "
|
|
f"pool_wait_max={max((p['max_waiting'] for p in summary['pools'].values()), default=0)}", flush=True)
|
|
return summary
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--base-url", default="http://127.0.0.1:18080")
|
|
parser.add_argument("--client", choices=("host", "container"), default="host")
|
|
parser.add_argument("--container-url", default="http://nginx")
|
|
parser.add_argument("--profiles", nargs="+", choices=("one", "two", "four"), default=["one", "two", "four"])
|
|
parser.add_argument("--rates", nargs="+", type=int, default=[100], help="Scheduled requests/s (default: 100 only)")
|
|
parser.add_argument("--duration-seconds", type=int, default=60, help="Measured phase per run (default: 60 seconds)")
|
|
parser.add_argument("--warmup-seconds", type=int, default=15, help="Read-only warmup per run (default: 15 seconds)")
|
|
parser.add_argument("--repetitions", type=int, default=3)
|
|
parser.add_argument("--full-matrix", action="store_true", help="Run all requested rates; record threshold crossings instead of stopping a profile")
|
|
parser.add_argument("--output", type=Path, required=True)
|
|
config = parser.parse_args()
|
|
if config.repetitions < 1 or config.duration_seconds < 1 or config.warmup_seconds < 0 or any(r < 1 or r > 1000 for r in config.rates):
|
|
parser.error("invalid duration, repetitions or rates")
|
|
config.output = config.output.resolve()
|
|
config.output.mkdir(parents=True, exist_ok=False)
|
|
manifest = {"started": datetime.now(timezone.utc).isoformat(), "profiles": config.profiles,
|
|
"client": config.client,
|
|
"rates": config.rates, "duration_seconds": config.duration_seconds,
|
|
"warmup_seconds": config.warmup_seconds, "repetitions": config.repetitions,
|
|
"full_matrix": config.full_matrix,
|
|
"stop_rule": ("record only; continue all requested rates after >=2 repeats exceed thresholds: " if config.full_matrix
|
|
else "stop a profile after >=2 repeats at one rate exceed any threshold: ") +
|
|
"scheduler late >5%, capacity drops >5%, request errors >1%, success p95 >1000ms",
|
|
"sampling": "API queue every 100ms; API JSON/cgroup about 1s; other services and PG about 2s",
|
|
"observer_version": 3,
|
|
"source_sha256": {name: hashlib.sha256((ROOT/name).read_bytes()).hexdigest() for name in (
|
|
"scripts/diagnostics/pool.cjs", "scripts/diagnostics/observer.sh",
|
|
"scripts/benchmark-scaling.mjs", "scripts/run-scaling-diagnostics.py")},
|
|
"notes": ["All profiles use the same instrumentation; actual per-replica pool limits are recorded in containers.",
|
|
"A container client uses the nginx service directly; it shares Docker VM CPU and bypasses Windows port forwarding.",
|
|
"Acquisition latency includes opening new connections; it is not SQL execution time.",
|
|
"Observers consume resources inside the existing container budgets.",
|
|
"Instrumentation stores no tokens, passwords, message text or SQL parameters."],
|
|
"containers": {}, "environments": {}, "runs": [], "stopped_profiles": {}, "threshold_crossings": {}}
|
|
manifest_path = config.output / "manifest.json"
|
|
save = lambda: manifest_path.write_text(json.dumps(manifest, indent=2) + "\n", encoding="utf-8")
|
|
save()
|
|
try:
|
|
for profile in config.profiles:
|
|
print(f"SWITCH {profile}", flush=True)
|
|
command(compose(profile) + ["up", "-d", "--no-build", "--no-deps", "--wait", "--wait-timeout", "120", "api"])
|
|
time.sleep(6) # Allow readiness's two successes and nginx reload to settle.
|
|
containers = {}
|
|
for role in ("api", "postgres", "nginx", "redis"):
|
|
containers[role] = command(compose(profile) + ["ps", "-q", role]).split()
|
|
if not containers[role]:
|
|
raise RuntimeError(f"no running {role}")
|
|
details = json.loads(command(["docker", "inspect", *[c for ids in containers.values() for c in ids]]))
|
|
running = [json.loads(line) for line in command(["docker", "ps", "--format", "{{json .}}"]).splitlines()]
|
|
manifest["environments"][profile] = {
|
|
"docker": json.loads(command(["docker", "info", "--format",
|
|
'{"cpus":{{.NCPU}},"memory_bytes":{{.MemTotal}},"version":{{json .ServerVersion}},"kernel":{{json .KernelVersion}}}'])),
|
|
"running_containers": [{"name": row.get("Names"), "image": row.get("Image")} for row in running],
|
|
}
|
|
manifest["containers"][profile] = [{"id": c["Id"], "name": c["Name"], "image": c["Image"],
|
|
"nano_cpus": c["HostConfig"]["NanoCpus"], "memory_bytes": c["HostConfig"]["Memory"],
|
|
"pool_max": next((e.split("=", 1)[1] for e in c["Config"]["Env"] if e.startswith("DB_POOL_MAX=")), None)} for c in details]
|
|
save()
|
|
for rate in config.rates:
|
|
batch = []
|
|
for repeat in range(1, config.repetitions + 1):
|
|
print(f"START {profile} rate={rate} repeat={repeat}", flush=True)
|
|
summary = run_one(config, profile, rate, repeat, containers)
|
|
batch.append(summary)
|
|
manifest["runs"].append(summary["label"])
|
|
save()
|
|
reasons = [key for key in batch[0]["limits_reached"]
|
|
if sum(r["limits_reached"][key] for r in batch) >= min(2, config.repetitions)]
|
|
if reasons:
|
|
manifest["threshold_crossings"].setdefault(profile, []).append({"rate": rate, "reasons": reasons})
|
|
if not config.full_matrix:
|
|
manifest["stopped_profiles"][profile] = {"rate": rate, "reasons": reasons}
|
|
print(f"{'CONTINUE (full matrix)' if config.full_matrix else 'STOP'} {profile} at {rate}: {', '.join(reasons)}", flush=True)
|
|
save()
|
|
if not config.full_matrix:
|
|
break
|
|
finally:
|
|
print("RESTORE two (without diagnostic preload)", flush=True)
|
|
command(compose("two", diagnostics=False) + ["up", "-d", "--no-build", "--no-deps", "--wait", "--wait-timeout", "120", "api"])
|
|
manifest["finished"] = datetime.now(timezone.utc).isoformat()
|
|
save()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|