#!/usr/bin/env python3 """Series A and B with a fixed total API RAM budget and the base 8/8/1 profile. The profile is applied through Compose environment, then checked with docker inspect before the load window. Nothing secret is printed. """ import json import os import subprocess import sys import threading import time import urllib.error import urllib.request from decimal import Decimal from pathlib import Path ROOT = Path(__file__).resolve().parents[1] HTTP_WORKERS = "8" DB_POOL_SIZE = "8" NGINX_WORKERS = "1" LB_CPU_LIMIT = "0.25" TOTAL_API_RAM = 512 * 1024 * 1024 def mem_bytes(spec): text = str(spec).strip() units = {"b": 1, "k": 1024, "m": 1024**2, "g": 1024**3} unit = text[-1].lower() if unit not in units: raise SystemExit(f"bad memory spec: {spec}") return int(text[:-1]) * units[unit] def cpu_nanos(spec): return int(Decimal(str(spec)) * Decimal(1_000_000_000)) def series_points(): points = [] for replicas, api_cpu in ((1, "1.0"), (2, "0.5"), (4, "0.25")): per = TOTAL_API_RAM // replicas // (1024 * 1024) points.append(("A", replicas, api_cpu, f"{per}m", "2.0")) for db_cpu in ("1.0", "2.0", "4.0"): per = TOTAL_API_RAM // 4 // (1024 * 1024) points.append(("B", 4, "0.25", f"{per}m", db_cpu)) return points def env_for(replicas, api_cpu, api_mem, db_cpu): env = os.environ.copy() env["API_REPLICAS"] = str(replicas) env["API_CPU_LIMIT"] = str(api_cpu) env["API_MEM_LIMIT"] = api_mem env["DB_CPU_LIMIT"] = str(db_cpu) env["HTTP_WORKERS"] = HTTP_WORKERS env["DB_POOL_SIZE"] = DB_POOL_SIZE env["NGINX_WORKERS"] = NGINX_WORKERS env["LB_CPU_LIMIT"] = LB_CPU_LIMIT return env def compose(args, env, capture=False): result = subprocess.run( ["docker", "compose", "--env-file", str(ROOT / ".env"), *args], cwd=ROOT, env=env, check=False, text=True, capture_output=capture, ) if result.returncode != 0 and capture: sys.stderr.write(result.stderr or "") sys.stderr.write(result.stdout or "") return result def wait_stack(env, replicas, timeout=180): deadline = time.monotonic() + timeout last = "" while time.monotonic() < deadline: ps = compose(["ps", "api", "--format", "{{.Health}}"], env, capture=True) health = [line.strip() for line in ps.stdout.splitlines() if line.strip()] ready = False try: with urllib.request.urlopen("http://127.0.0.1:8080/health/ready", timeout=2) as resp: ready = resp.status == 200 except (urllib.error.URLError, TimeoutError, OSError): ready = False last = f"api_health={health} ready={ready}" if len(health) == replicas and all(item == "healthy" for item in health) and ready: return time.sleep(2) raise SystemExit("stack did not become ready: " + last) def container_ids(env, service): result = compose(["ps", "-q", service], env, capture=True) if result.returncode != 0: raise SystemExit(f"compose ps {service} failed") return [line for line in result.stdout.split() if line] def inspect_container(cid): raw = subprocess.check_output(["docker", "inspect", cid], text=True) return json.loads(raw)[0] def env_map(info): found = {} for item in info["Config"].get("Env") or []: key, _, value = item.partition("=") found[key] = value return found def verify_limits(env, replicas, api_cpu, api_mem, db_cpu): expect_api_nanos = cpu_nanos(api_cpu) expect_api_mem = mem_bytes(api_mem) expect_db_nanos = cpu_nanos(db_cpu) expect_lb_nanos = cpu_nanos(LB_CPU_LIMIT) api_ids = container_ids(env, "api") if len(api_ids) != replicas: raise SystemExit(f"expected {replicas} api containers, found {len(api_ids)}") api_rows = [] for cid in api_ids: info = inspect_container(cid) nanos = info["HostConfig"]["NanoCpus"] memory = info["HostConfig"]["Memory"] present = env_map(info) name = info["Name"].lstrip("/") if nanos != expect_api_nanos or memory != expect_api_mem: raise SystemExit( f"{name}: NanoCpus={nanos} Memory={memory}, " f"expected {expect_api_nanos} and {expect_api_mem}" ) if present.get("HTTP_WORKERS") != HTTP_WORKERS or present.get("DB_POOL_SIZE") != DB_POOL_SIZE: raise SystemExit( f"{name}: HTTP_WORKERS={present.get('HTTP_WORKERS')} " f"DB_POOL_SIZE={present.get('DB_POOL_SIZE')}" ) api_rows.append( { "name": name, "nano_cpus": nanos, "memory": memory, "http_workers": present["HTTP_WORKERS"], "db_pool_size": present["DB_POOL_SIZE"], } ) total_mem = sum(row["memory"] for row in api_rows) if total_mem != TOTAL_API_RAM: raise SystemExit(f"api RAM sum {total_mem} != {TOTAL_API_RAM}") db_ids = container_ids(env, "db") if len(db_ids) != 1: raise SystemExit(f"expected 1 db container, found {len(db_ids)}") db_info = inspect_container(db_ids[0]) db_nanos = db_info["HostConfig"]["NanoCpus"] if db_nanos != expect_db_nanos: raise SystemExit(f"db NanoCpus={db_nanos}, expected {expect_db_nanos}") lb_ids = container_ids(env, "lb") if len(lb_ids) != 1: raise SystemExit(f"expected 1 lb container, found {len(lb_ids)}") lb_info = inspect_container(lb_ids[0]) lb_nanos = lb_info["HostConfig"]["NanoCpus"] lb_env = env_map(lb_info) if lb_nanos != expect_lb_nanos or lb_env.get("NGINX_WORKERS") != NGINX_WORKERS: raise SystemExit( f"lb NanoCpus={lb_nanos} NGINX_WORKERS={lb_env.get('NGINX_WORKERS')}" ) rendered = compose(["exec", "-T", "lb", "nginx", "-T"], env, capture=True) if rendered.returncode != 0: raise SystemExit("nginx -T failed") directive = "" rendered_text = (rendered.stdout or "") + "\n" + (rendered.stderr or "") for line in rendered_text.splitlines(): stripped = line.strip() if stripped.startswith("worker_processes"): directive = stripped break expected_directive = f"worker_processes {NGINX_WORKERS};" if directive != expected_directive: raise SystemExit(f"nginx config has {directive!r}, expected {expected_directive!r}") logs = compose(["logs", "api"], env, capture=True) if logs.returncode != 0: raise SystemExit("api logs failed") pool_line = f"pg pool ready ({DB_POOL_SIZE} connections)" worker_line = f"http workers: {HTTP_WORKERS}" if logs.stdout.count(pool_line) < replicas or logs.stdout.count(worker_line) < replicas: raise SystemExit("api logs do not show the base pool and worker profile on every replica") return { "verified": True, "api": api_rows, "api_memory_sum": total_mem, "db": {"name": db_info["Name"].lstrip("/"), "nano_cpus": db_nanos}, "lb": { "name": lb_info["Name"].lstrip("/"), "nano_cpus": lb_nanos, "nginx_worker_processes": directive, }, } def sample_stats(stop, bucket): while not stop.is_set(): proc = subprocess.run( ["docker", "stats", "--no-stream", "--format", "{{.Name}}\t{{.CPUPerc}}\t{{.MemUsage}}"], text=True, capture_output=True, check=False, ) stamp = time.time() rows = [] for line in proc.stdout.splitlines(): parts = line.split("\t") if len(parts) != 3: continue name, cpu, mem = parts try: cpu_num = float(cpu.strip().rstrip("%")) except ValueError: continue rows.append({"name": name, "cpu_percent": cpu_num, "mem": mem.strip()}) if rows: bucket.append({"t": stamp, "rows": rows}) stop.wait(3) def summarize_cpu(samples): acc = {} for sample in samples: for row in sample["rows"]: name = row["name"].lstrip("/") service = "other" for prefix in ("api", "db", "cache", "lb"): if f"-{prefix}-" in name or name.endswith(prefix): service = prefix break acc.setdefault(service, []).append(row["cpu_percent"]) out = {} for service, values in acc.items(): out[service] = { "samples": len(values), "mean_cpu_percent": round(sum(values) / len(values), 2), "max_cpu_percent": round(max(values), 2), } return out def run_point(series, replicas, api_cpu, api_mem, db_cpu): env = env_for(replicas, api_cpu, api_mem, db_cpu) label = f"track1-{series}-r{replicas}-apicpu{api_cpu}-dbcpu{db_cpu}" raw = ROOT / "evidence" / f"{label}.load.json" dest = ROOT / "evidence" / f"{label}.json" print( f"point series={series} replicas={replicas} api_cpu={api_cpu} " f"api_mem={api_mem} db_cpu={db_cpu} http_workers={HTTP_WORKERS} " f"db_pool={DB_POOL_SIZE} nginx_workers={NGINX_WORKERS}", file=sys.stderr, ) down = compose(["down"], env) if down.returncode != 0: raise SystemExit(down.returncode) up = compose( ["up", "-d", "--build", "--scale", f"api={replicas}"], env, ) if up.returncode != 0: raise SystemExit(up.returncode) wait_stack(env, replicas) limits = verify_limits(env, replicas, api_cpu, api_mem, db_cpu) print( f"inspect ok api_sum_memory={limits['api_memory_sum']} " f"api_nano={limits['api'][0]['nano_cpus']} db_nano={limits['db']['nano_cpus']} " f"nginx={limits['lb']['nginx_worker_processes']}", file=sys.stderr, ) warmup = subprocess.run( [ sys.executable, str(ROOT / "tests" / "load.py"), "--duration", "10", "--concurrency", "50", "--output", f"/tmp/{label}-warmup.json", ], cwd=ROOT, text=True, capture_output=True, ) if warmup.returncode != 0: sys.stderr.write(warmup.stdout) sys.stderr.write(warmup.stderr) raise SystemExit("warmup failed") samples = [] stop = threading.Event() thread = threading.Thread(target=sample_stats, args=(stop, samples)) thread.start() try: measured = subprocess.run( [ sys.executable, str(ROOT / "tests" / "load.py"), "--duration", "30", "--concurrency", "50", "--output", str(raw), ], cwd=ROOT, text=True, capture_output=True, ) finally: stop.set() thread.join() if measured.returncode != 0: sys.stderr.write(measured.stdout) sys.stderr.write(measured.stderr) raise SystemExit("measured run failed") load = json.loads(raw.read_text()) report = { "series": series, "api_replicas": replicas, "api_cpu_limit": float(api_cpu), "api_mem_limit": api_mem, "api_mem_limit_bytes": mem_bytes(api_mem), "api_mem_total_bytes": TOTAL_API_RAM, "db_cpu_limit": float(db_cpu), "http_workers": int(HTTP_WORKERS), "pg_pool": int(DB_POOL_SIZE), "nginx_worker_processes": NGINX_WORKERS, "lb_cpu_limit": float(LB_CPU_LIMIT), "warmup_seconds": 10, "inspect": limits, "cpu": summarize_cpu(samples), "load": load, } dest.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n") lat = load["successful_latency_ms"] summary = { "file": str(dest.relative_to(ROOT)), "rps": load["successful_rps"], "p50": lat["p50"], "p95": lat["p95"], "p99": lat["p99"], "errors": load["failed_requests"], "error_ratio": load["error_ratio"], "api_mem_total_bytes": TOTAL_API_RAM, "cpu": report["cpu"], } print(json.dumps(summary, ensure_ascii=False)) return summary def main(): if len(sys.argv) != 1: raise SystemExit("usage: python3 scripts/run_track1.py") for series, replicas, api_cpu, api_mem, db_cpu in series_points(): run_point(series, replicas, api_cpu, api_mem, db_cpu) if __name__ == "__main__": main()