#!/usr/bin/env python3 """Run a bounded scaling sweep and collect diagnostics without HTTP metric endpoints.""" import argparse from datetime import datetime, timezone import json from pathlib import Path import subprocess import time import uuid import hashlib ROOT = Path(__file__).resolve().parents[1] def command(args, timeout=180): result = subprocess.run(args, cwd=ROOT, capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=timeout) if result.returncode: raise RuntimeError(f"{args[0]} {args[1]} failed: {result.stderr[-1500:]}") return result.stdout def compose(profile, diagnostics=True): args = ["docker", "compose", "-f", "compose.yaml", "-f", f"configs/{profile}.yaml"] if diagnostics: args += ["-f", "test/compose.diagnostics.yaml"] return args def rows(path): data = [] if path.exists(): text = path.read_text(encoding="utf-8") decoder = json.JSONDecoder() position = 0 while position < len(text): if text[position].isspace(): position += 1 continue try: value, position = decoder.raw_decode(text, position) data.append(value) except json.JSONDecodeError: break # Last record can be in progress when a running observer is copied. return data def delta(first, last, name): return max(0, last.get(name, 0) - first.get(name, 0)) def resource_summary(samples): if len(samples) < 2: return {"samples": len(samples)} first, last = samples[0], samples[-1] elapsed = (last["time_ms"] - first["time_ms"]) / 1000 periods = delta(first, last, "cpu_nr_periods") return { "samples": len(samples), "coverage_seconds": elapsed, "max_sample_gap_seconds": max((b["time_ms"] - a["time_ms"]) / 1000 for a, b in zip(samples, samples[1:])), "mean_cpu_cores": delta(first, last, "cpu_usage_usec") / max(elapsed * 1e6, 1), "throttled_seconds": delta(first, last, "cpu_throttled_usec") / 1e6, "throttled_period_fraction": delta(first, last, "cpu_nr_throttled") / max(periods, 1), "peak_memory_mib": max(s.get("memory_bytes", 0) or 0 for s in samples) / 1024**2, "oom_kills": delta(first, last, "memory_oom_kill"), } def summarize(directory, benchmark): start = benchmark["timing"]["measurement_started_epoch_ms"] end = benchmark["timing"]["measurement_scheduled_end_epoch_ms"] within = lambda data: [r for r in data if start <= r["time_ms"] <= end] resources = {} pools = {} for path in sorted(directory.glob("api-*.jsonl")): data = within(rows(path)) flattened = [{**r, **{f"cpu_{k}": v for k, v in (r.get("cpu") or {}).items()}, **{f"memory_{k}": v for k, v in (r.get("memory_events") or {}).items()}} for r in data] resources[path.stem] = resource_summary(flattened) if len(data) < 2: continue first, last = data[0]["pools"], data[-1]["pools"] acquisitions = sum(p["acquisitions"] for p in last) - sum(p["acquisitions"] for p in first) acquire_ms = sum(p["acquire_ms"] for p in last) - sum(p["acquire_ms"] for p in first) intervals = [p for r in data for p in r["pools"]] totals = [sum(p["total"] for p in r["pools"]) for r in data] pools[path.stem] = { "samples": len(data), "acquisitions": acquisitions, "acquisition_errors": sum(p["errors"] for p in last) - sum(p["errors"] for p in first), "mean_acquisition_ms": acquire_ms / max(acquisitions, 1), "max_acquisition_ms": max((p["interval_max_ms"] for p in intervals), default=0), "max_waiting": max((p["interval_max_waiting"] for p in intervals), default=0), "queued_interval_fraction": sum(p["interval_max_waiting"] > 0 for p in intervals) / max(len(intervals), 1), "max_total": max((p["total"] for p in intervals), default=0), "observed_connection_increases": sum(max(0, b-a) for a, b in zip(totals, totals[1:])), "observed_connection_decreases": sum(max(0, a-b) for a, b in zip(totals, totals[1:])), "event_loop_p99_max_ms": max(r["event_loop_delay_ms"]["p99"] for r in data), } pg = within(rows(directory / "postgres.jsonl")) pg_samples = [r for r in pg if r["kind"] == "postgres"] pg_samples += within(rows(directory / "postgres-sql.jsonl")) pg_samples.sort(key=lambda r: r["time_ms"]) host_samples = [r for r in pg if r["kind"] == "host"] host = {"samples": len(host_samples)} if len(host_samples) > 1: host_seconds = (host_samples[-1]["time_ms"] - host_samples[0]["time_ms"])/1000 host.update({key + "_fraction": delta(host_samples[0], host_samples[-1], key + "_total_usec") / max(host_seconds*1e6, 1) for key in ("cpu_some", "memory_some", "memory_full", "io_some", "io_full")}) host["min_available_mib"] = min(r["MemAvailable_kib"] for r in host_samples)/1024 host["max_swap_used_mib"] = max(r["SwapTotal_kib"] - r["SwapFree_kib"] for r in host_samples)/1024 waits = {} max_active = max_idle_tx = 0 for sample in pg_samples: max_active = max(max_active, sum(a["count"] for a in sample["activity"] if a["state"] == "active")) max_idle_tx = max(max_idle_tx, sum(a["count"] for a in sample["activity"] if a["state"] == "idle in transaction")) for a in sample["activity"]: if a["state"] == "active": key = f"{a['wait_event_type'] or 'no_wait_event'}/{a['wait_event'] or '-'}" waits[key] = waits.get(key, 0) + a["count"] for role in ("postgres", "nginx", "redis"): resources[role] = resource_summary([r for r in within(rows(directory / f"{role}.jsonl")) if r["kind"] == "cgroup"]) upstream = [r for r in within(rows(directory / "nginx.jsonl")) if r["kind"] == "upstream"] changes = sum(a["servers"] != b["servers"] for a, b in zip(upstream, upstream[1:])) m = benchmark["measurement"] return { "label": benchmark["label"], "profile": benchmark["profile"], "measurement": m, "client": benchmark["client"], "resources": resources, "pools": pools, "host": host, "postgres": {"samples": len(pg_samples), "max_active": max_active, "max_idle_in_transaction": max_idle_tx, "active_wait_observations": waits}, "nginx": {"samples": len(upstream), "upstream_changes": changes, "min_ready": min((len(r["servers"]) for r in upstream), default=None), "max_ready": max((len(r["servers"]) for r in upstream), default=None)}, "limits_reached": { "generator_late": m["scheduler_late_slots"] / m["scheduled_slots"] > 0.05, "client_capacity": (m["client_dropped_slots"] - m["scheduler_late_slots"]) / m["scheduled_slots"] > 0.05, "request_errors": m["all"]["errors"] / max(m["all"]["attempted"], 1) > 0.01, "latency": (m["all"]["successful_latency_ms"]["p95"] or 0) > 1000, }, } def run_one(config, profile, rate, repeat, containers): label = f"{profile}-r{rate}-repeat{repeat}" directory = config.output / label directory.mkdir(parents=True, exist_ok=False) workers = [] stop = f"/tmp/scaling-observer-{uuid.uuid4().hex}.stop" observer = (ROOT / "scripts/diagnostics/observer.sh").read_bytes() client_container = None try: for role in ("postgres", "nginx", "redis"): output = (directory / f"{role}.jsonl").open("wb") errors = (directory / f"{role}.stderr").open("wb") child = subprocess.Popen(["docker", "exec", "-i", containers[role][0], "sh", "-s", role, stop], cwd=ROOT, stdin=subprocess.PIPE, stdout=output, stderr=errors) workers.append((role, child, output, errors)) child.stdin.write(observer) child.stdin.close() started = datetime.now(timezone.utc).isoformat() target = directory / "benchmark.json" args = ["node", "scripts/benchmark-scaling.mjs", "--base-url", config.base_url, "--label", label, "--rate", str(rate), "--duration-seconds", str(config.duration_seconds), "--warmup-seconds", str(config.warmup_seconds), "--seed-messages", "30", "--expected-instances", str({"one": 1, "two": 2, "four": 4}[profile]), "--output", str(target)] if config.client == "container": nginx = json.loads(command(["docker", "inspect", containers["nginx"][0]]))[0] api = json.loads(command(["docker", "inspect", containers["api"][0]]))[0] networks = set(nginx["NetworkSettings"]["Networks"]) & set(api["NetworkSettings"]["Networks"]) if len(networks) != 1: raise RuntimeError("expected one common API/nginx network") client_container = "lab3-diagnostic-client-" + uuid.uuid4().hex args[args.index("--base-url") + 1] = config.container_url args[args.index("--output") + 1] = "/tmp/scaling-benchmark.json" args[1] = "/benchmark.mjs" args = ["docker", "run", "--name", client_container, "--no-healthcheck", "--network", next(iter(networks)), "-v", f"{ROOT / 'scripts/benchmark-scaling.mjs'}:/benchmark.mjs:ro", "--entrypoint", "node", api["Image"], *args[1:]] result = subprocess.run(args, cwd=ROOT, capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=config.duration_seconds + config.warmup_seconds + 120) (directory / "benchmark.stderr").write_text(result.stderr, encoding="utf-8") if client_container is not None: command(["docker", "cp", f"{client_container}:/tmp/scaling-benchmark.json", str(target)]) if not target.exists(): raise RuntimeError(f"{label}: benchmark did not produce results: {result.stderr}") benchmark = json.loads(target.read_text(encoding="utf-8")) for index, container in enumerate(containers["api"], 1): command(["docker", "cp", f"{container}:/tmp/scaling-pool.jsonl", str(directory / f"api-{index}.jsonl")]) ended = datetime.now(timezone.utc).isoformat() # Docker puts nginx stderr notices on CLI stderr; collect both streams. log_result = subprocess.run(["docker", "logs", "--timestamps", "--since", started, "--until", ended, containers["nginx"][0]], cwd=ROOT, capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=30) diagnostic_lines = [line for line in (log_result.stdout + log_result.stderr).splitlines() if any(key in line for key in ("signal process started", "download timed out", "no live upstreams", "upstream timed out"))] (directory / "nginx-events.log").write_text("\n".join(diagnostic_lines), encoding="utf-8") (directory / "execution.json").write_text(json.dumps({"returncode": result.returncode, "started": started, "ended": ended, "command": args}, indent=2), encoding="utf-8") finally: if client_container is not None: # Only the uniquely named disposable generator created by this run. subprocess.run(["docker", "rm", "-f", client_container], cwd=ROOT, capture_output=True, timeout=30) for role, child, output, errors in workers: try: command(["docker", "exec", containers[role][0], "touch", stop], timeout=15) child.wait(timeout=15) except (RuntimeError, subprocess.TimeoutExpired): child.terminate() child.wait(timeout=10) output.close() errors.close() if role == "postgres": command(["docker", "cp", f"{containers[role][0]}:{stop}.pg.jsonl", str(directory / "postgres-sql.jsonl")]) command(["docker", "exec", containers[role][0], "rm", "-f", stop + ".pg.jsonl"]) summary = summarize(directory, benchmark) summary["diagnostics_complete"] = ( len(summary["pools"]) == len(containers["api"]) and summary["postgres"]["samples"] > 0 and summary["nginx"]["samples"] > 0 and all(v["samples"] > 0 for v in summary["resources"].values()) ) (directory / "summary.json").write_text(json.dumps(summary, indent=2) + "\n", encoding="utf-8") if not summary["diagnostics_complete"]: raise RuntimeError(f"{label}: diagnostics incomplete; inspect observer stderr files") m = summary["measurement"] print(f"DONE {label}: RPS={m['all']['successful_rps']} p95={m['all']['successful_latency_ms']['p95']}ms " f"errors={m['all']['errors']} drops={m['client_dropped_slots']}/{m['scheduled_slots']} " f"pool_wait_max={max((p['max_waiting'] for p in summary['pools'].values()), default=0)}", flush=True) return summary def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--base-url", default="http://127.0.0.1:18080") parser.add_argument("--client", choices=("host", "container"), default="host") parser.add_argument("--container-url", default="http://nginx") parser.add_argument("--profiles", nargs="+", choices=("one", "two", "four"), default=["one", "two", "four"]) parser.add_argument("--rates", nargs="+", type=int, default=[100], help="Scheduled requests/s (default: 100 only)") parser.add_argument("--duration-seconds", type=int, default=60, help="Measured phase per run (default: 60 seconds)") parser.add_argument("--warmup-seconds", type=int, default=15, help="Read-only warmup per run (default: 15 seconds)") parser.add_argument("--repetitions", type=int, default=3) parser.add_argument("--full-matrix", action="store_true", help="Run all requested rates; record threshold crossings instead of stopping a profile") parser.add_argument("--output", type=Path, required=True) config = parser.parse_args() if config.repetitions < 1 or config.duration_seconds < 1 or config.warmup_seconds < 0 or any(r < 1 or r > 1000 for r in config.rates): parser.error("invalid duration, repetitions or rates") config.output = config.output.resolve() config.output.mkdir(parents=True, exist_ok=False) manifest = {"started": datetime.now(timezone.utc).isoformat(), "profiles": config.profiles, "client": config.client, "rates": config.rates, "duration_seconds": config.duration_seconds, "warmup_seconds": config.warmup_seconds, "repetitions": config.repetitions, "full_matrix": config.full_matrix, "stop_rule": ("record only; continue all requested rates after >=2 repeats exceed thresholds: " if config.full_matrix else "stop a profile after >=2 repeats at one rate exceed any threshold: ") + "scheduler late >5%, capacity drops >5%, request errors >1%, success p95 >1000ms", "sampling": "API queue every 100ms; API JSON/cgroup about 1s; other services and PG about 2s", "observer_version": 3, "source_sha256": {name: hashlib.sha256((ROOT/name).read_bytes()).hexdigest() for name in ( "scripts/diagnostics/pool.cjs", "scripts/diagnostics/observer.sh", "scripts/benchmark-scaling.mjs", "scripts/run-scaling-diagnostics.py")}, "notes": ["All profiles use the same instrumentation; actual per-replica pool limits are recorded in containers.", "A container client uses the nginx service directly; it shares Docker VM CPU and bypasses Windows port forwarding.", "Acquisition latency includes opening new connections; it is not SQL execution time.", "Observers consume resources inside the existing container budgets.", "Instrumentation stores no tokens, passwords, message text or SQL parameters."], "containers": {}, "environments": {}, "runs": [], "stopped_profiles": {}, "threshold_crossings": {}} manifest_path = config.output / "manifest.json" save = lambda: manifest_path.write_text(json.dumps(manifest, indent=2) + "\n", encoding="utf-8") save() try: for profile in config.profiles: print(f"SWITCH {profile}", flush=True) command(compose(profile) + ["up", "-d", "--no-build", "--no-deps", "--wait", "--wait-timeout", "120", "api"]) time.sleep(6) # Allow readiness's two successes and nginx reload to settle. containers = {} for role in ("api", "postgres", "nginx", "redis"): containers[role] = command(compose(profile) + ["ps", "-q", role]).split() if not containers[role]: raise RuntimeError(f"no running {role}") details = json.loads(command(["docker", "inspect", *[c for ids in containers.values() for c in ids]])) running = [json.loads(line) for line in command(["docker", "ps", "--format", "{{json .}}"]).splitlines()] manifest["environments"][profile] = { "docker": json.loads(command(["docker", "info", "--format", '{"cpus":{{.NCPU}},"memory_bytes":{{.MemTotal}},"version":{{json .ServerVersion}},"kernel":{{json .KernelVersion}}}'])), "running_containers": [{"name": row.get("Names"), "image": row.get("Image")} for row in running], } manifest["containers"][profile] = [{"id": c["Id"], "name": c["Name"], "image": c["Image"], "nano_cpus": c["HostConfig"]["NanoCpus"], "memory_bytes": c["HostConfig"]["Memory"], "pool_max": next((e.split("=", 1)[1] for e in c["Config"]["Env"] if e.startswith("DB_POOL_MAX=")), None)} for c in details] save() for rate in config.rates: batch = [] for repeat in range(1, config.repetitions + 1): print(f"START {profile} rate={rate} repeat={repeat}", flush=True) summary = run_one(config, profile, rate, repeat, containers) batch.append(summary) manifest["runs"].append(summary["label"]) save() reasons = [key for key in batch[0]["limits_reached"] if sum(r["limits_reached"][key] for r in batch) >= min(2, config.repetitions)] if reasons: manifest["threshold_crossings"].setdefault(profile, []).append({"rate": rate, "reasons": reasons}) if not config.full_matrix: manifest["stopped_profiles"][profile] = {"rate": rate, "reasons": reasons} print(f"{'CONTINUE (full matrix)' if config.full_matrix else 'STOP'} {profile} at {rate}: {', '.join(reasons)}", flush=True) save() if not config.full_matrix: break finally: print("RESTORE two (without diagnostic preload)", flush=True) command(compose("two", diagnostics=False) + ["up", "-d", "--no-build", "--no-deps", "--wait", "--wait-timeout", "120", "api"]) manifest["finished"] = datetime.now(timezone.utc).isoformat() save() if __name__ == "__main__": main()