forked from mtusi2026/mtusi2026lab3
375 lines
12 KiB
Python
Executable File
375 lines
12 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Series A and B with a fixed total API RAM budget and the base 8/8/1 profile.
|
|
|
|
The profile is applied through Compose environment, then checked with docker inspect
|
|
before the load window. Nothing secret is printed.
|
|
"""
|
|
import json
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
import threading
|
|
import time
|
|
import urllib.error
|
|
import urllib.request
|
|
from decimal import Decimal
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
HTTP_WORKERS = "8"
|
|
DB_POOL_SIZE = "8"
|
|
NGINX_WORKERS = "1"
|
|
LB_CPU_LIMIT = "0.25"
|
|
TOTAL_API_RAM = 512 * 1024 * 1024
|
|
|
|
|
|
def mem_bytes(spec):
|
|
text = str(spec).strip()
|
|
units = {"b": 1, "k": 1024, "m": 1024**2, "g": 1024**3}
|
|
unit = text[-1].lower()
|
|
if unit not in units:
|
|
raise SystemExit(f"bad memory spec: {spec}")
|
|
return int(text[:-1]) * units[unit]
|
|
|
|
|
|
def cpu_nanos(spec):
|
|
return int(Decimal(str(spec)) * Decimal(1_000_000_000))
|
|
|
|
|
|
def series_points():
|
|
points = []
|
|
for replicas, api_cpu in ((1, "1.0"), (2, "0.5"), (4, "0.25")):
|
|
per = TOTAL_API_RAM // replicas // (1024 * 1024)
|
|
points.append(("A", replicas, api_cpu, f"{per}m", "2.0"))
|
|
for db_cpu in ("1.0", "2.0", "4.0"):
|
|
per = TOTAL_API_RAM // 4 // (1024 * 1024)
|
|
points.append(("B", 4, "0.25", f"{per}m", db_cpu))
|
|
return points
|
|
|
|
|
|
def env_for(replicas, api_cpu, api_mem, db_cpu):
|
|
env = os.environ.copy()
|
|
env["API_REPLICAS"] = str(replicas)
|
|
env["API_CPU_LIMIT"] = str(api_cpu)
|
|
env["API_MEM_LIMIT"] = api_mem
|
|
env["DB_CPU_LIMIT"] = str(db_cpu)
|
|
env["HTTP_WORKERS"] = HTTP_WORKERS
|
|
env["DB_POOL_SIZE"] = DB_POOL_SIZE
|
|
env["NGINX_WORKERS"] = NGINX_WORKERS
|
|
env["LB_CPU_LIMIT"] = LB_CPU_LIMIT
|
|
return env
|
|
|
|
|
|
def compose(args, env, capture=False):
|
|
result = subprocess.run(
|
|
["docker", "compose", "--env-file", str(ROOT / ".env"), *args],
|
|
cwd=ROOT,
|
|
env=env,
|
|
check=False,
|
|
text=True,
|
|
capture_output=capture,
|
|
)
|
|
if result.returncode != 0 and capture:
|
|
sys.stderr.write(result.stderr or "")
|
|
sys.stderr.write(result.stdout or "")
|
|
return result
|
|
|
|
|
|
def wait_stack(env, replicas, timeout=180):
|
|
deadline = time.monotonic() + timeout
|
|
last = ""
|
|
while time.monotonic() < deadline:
|
|
ps = compose(["ps", "api", "--format", "{{.Health}}"], env, capture=True)
|
|
health = [line.strip() for line in ps.stdout.splitlines() if line.strip()]
|
|
ready = False
|
|
try:
|
|
with urllib.request.urlopen("http://127.0.0.1:8080/health/ready", timeout=2) as resp:
|
|
ready = resp.status == 200
|
|
except (urllib.error.URLError, TimeoutError, OSError):
|
|
ready = False
|
|
last = f"api_health={health} ready={ready}"
|
|
if len(health) == replicas and all(item == "healthy" for item in health) and ready:
|
|
return
|
|
time.sleep(2)
|
|
raise SystemExit("stack did not become ready: " + last)
|
|
|
|
|
|
def container_ids(env, service):
|
|
result = compose(["ps", "-q", service], env, capture=True)
|
|
if result.returncode != 0:
|
|
raise SystemExit(f"compose ps {service} failed")
|
|
return [line for line in result.stdout.split() if line]
|
|
|
|
|
|
def inspect_container(cid):
|
|
raw = subprocess.check_output(["docker", "inspect", cid], text=True)
|
|
return json.loads(raw)[0]
|
|
|
|
|
|
def env_map(info):
|
|
found = {}
|
|
for item in info["Config"].get("Env") or []:
|
|
key, _, value = item.partition("=")
|
|
found[key] = value
|
|
return found
|
|
|
|
|
|
def verify_limits(env, replicas, api_cpu, api_mem, db_cpu):
|
|
expect_api_nanos = cpu_nanos(api_cpu)
|
|
expect_api_mem = mem_bytes(api_mem)
|
|
expect_db_nanos = cpu_nanos(db_cpu)
|
|
expect_lb_nanos = cpu_nanos(LB_CPU_LIMIT)
|
|
api_ids = container_ids(env, "api")
|
|
if len(api_ids) != replicas:
|
|
raise SystemExit(f"expected {replicas} api containers, found {len(api_ids)}")
|
|
|
|
api_rows = []
|
|
for cid in api_ids:
|
|
info = inspect_container(cid)
|
|
nanos = info["HostConfig"]["NanoCpus"]
|
|
memory = info["HostConfig"]["Memory"]
|
|
present = env_map(info)
|
|
name = info["Name"].lstrip("/")
|
|
if nanos != expect_api_nanos or memory != expect_api_mem:
|
|
raise SystemExit(
|
|
f"{name}: NanoCpus={nanos} Memory={memory}, "
|
|
f"expected {expect_api_nanos} and {expect_api_mem}"
|
|
)
|
|
if present.get("HTTP_WORKERS") != HTTP_WORKERS or present.get("DB_POOL_SIZE") != DB_POOL_SIZE:
|
|
raise SystemExit(
|
|
f"{name}: HTTP_WORKERS={present.get('HTTP_WORKERS')} "
|
|
f"DB_POOL_SIZE={present.get('DB_POOL_SIZE')}"
|
|
)
|
|
api_rows.append(
|
|
{
|
|
"name": name,
|
|
"nano_cpus": nanos,
|
|
"memory": memory,
|
|
"http_workers": present["HTTP_WORKERS"],
|
|
"db_pool_size": present["DB_POOL_SIZE"],
|
|
}
|
|
)
|
|
total_mem = sum(row["memory"] for row in api_rows)
|
|
if total_mem != TOTAL_API_RAM:
|
|
raise SystemExit(f"api RAM sum {total_mem} != {TOTAL_API_RAM}")
|
|
|
|
db_ids = container_ids(env, "db")
|
|
if len(db_ids) != 1:
|
|
raise SystemExit(f"expected 1 db container, found {len(db_ids)}")
|
|
db_info = inspect_container(db_ids[0])
|
|
db_nanos = db_info["HostConfig"]["NanoCpus"]
|
|
if db_nanos != expect_db_nanos:
|
|
raise SystemExit(f"db NanoCpus={db_nanos}, expected {expect_db_nanos}")
|
|
|
|
lb_ids = container_ids(env, "lb")
|
|
if len(lb_ids) != 1:
|
|
raise SystemExit(f"expected 1 lb container, found {len(lb_ids)}")
|
|
lb_info = inspect_container(lb_ids[0])
|
|
lb_nanos = lb_info["HostConfig"]["NanoCpus"]
|
|
lb_env = env_map(lb_info)
|
|
if lb_nanos != expect_lb_nanos or lb_env.get("NGINX_WORKERS") != NGINX_WORKERS:
|
|
raise SystemExit(
|
|
f"lb NanoCpus={lb_nanos} NGINX_WORKERS={lb_env.get('NGINX_WORKERS')}"
|
|
)
|
|
rendered = compose(["exec", "-T", "lb", "nginx", "-T"], env, capture=True)
|
|
if rendered.returncode != 0:
|
|
raise SystemExit("nginx -T failed")
|
|
directive = ""
|
|
rendered_text = (rendered.stdout or "") + "\n" + (rendered.stderr or "")
|
|
for line in rendered_text.splitlines():
|
|
stripped = line.strip()
|
|
if stripped.startswith("worker_processes"):
|
|
directive = stripped
|
|
break
|
|
expected_directive = f"worker_processes {NGINX_WORKERS};"
|
|
if directive != expected_directive:
|
|
raise SystemExit(f"nginx config has {directive!r}, expected {expected_directive!r}")
|
|
|
|
logs = compose(["logs", "api"], env, capture=True)
|
|
if logs.returncode != 0:
|
|
raise SystemExit("api logs failed")
|
|
pool_line = f"pg pool ready ({DB_POOL_SIZE} connections)"
|
|
worker_line = f"http workers: {HTTP_WORKERS}"
|
|
if logs.stdout.count(pool_line) < replicas or logs.stdout.count(worker_line) < replicas:
|
|
raise SystemExit("api logs do not show the base pool and worker profile on every replica")
|
|
|
|
return {
|
|
"verified": True,
|
|
"api": api_rows,
|
|
"api_memory_sum": total_mem,
|
|
"db": {"name": db_info["Name"].lstrip("/"), "nano_cpus": db_nanos},
|
|
"lb": {
|
|
"name": lb_info["Name"].lstrip("/"),
|
|
"nano_cpus": lb_nanos,
|
|
"nginx_worker_processes": directive,
|
|
},
|
|
}
|
|
|
|
|
|
def sample_stats(stop, bucket):
|
|
while not stop.is_set():
|
|
proc = subprocess.run(
|
|
["docker", "stats", "--no-stream", "--format", "{{.Name}}\t{{.CPUPerc}}\t{{.MemUsage}}"],
|
|
text=True,
|
|
capture_output=True,
|
|
check=False,
|
|
)
|
|
stamp = time.time()
|
|
rows = []
|
|
for line in proc.stdout.splitlines():
|
|
parts = line.split("\t")
|
|
if len(parts) != 3:
|
|
continue
|
|
name, cpu, mem = parts
|
|
try:
|
|
cpu_num = float(cpu.strip().rstrip("%"))
|
|
except ValueError:
|
|
continue
|
|
rows.append({"name": name, "cpu_percent": cpu_num, "mem": mem.strip()})
|
|
if rows:
|
|
bucket.append({"t": stamp, "rows": rows})
|
|
stop.wait(3)
|
|
|
|
|
|
def summarize_cpu(samples):
|
|
acc = {}
|
|
for sample in samples:
|
|
for row in sample["rows"]:
|
|
name = row["name"].lstrip("/")
|
|
service = "other"
|
|
for prefix in ("api", "db", "cache", "lb"):
|
|
if f"-{prefix}-" in name or name.endswith(prefix):
|
|
service = prefix
|
|
break
|
|
acc.setdefault(service, []).append(row["cpu_percent"])
|
|
out = {}
|
|
for service, values in acc.items():
|
|
out[service] = {
|
|
"samples": len(values),
|
|
"mean_cpu_percent": round(sum(values) / len(values), 2),
|
|
"max_cpu_percent": round(max(values), 2),
|
|
}
|
|
return out
|
|
|
|
|
|
def run_point(series, replicas, api_cpu, api_mem, db_cpu):
|
|
env = env_for(replicas, api_cpu, api_mem, db_cpu)
|
|
label = f"track1-{series}-r{replicas}-apicpu{api_cpu}-dbcpu{db_cpu}"
|
|
raw = ROOT / "evidence" / f"{label}.load.json"
|
|
dest = ROOT / "evidence" / f"{label}.json"
|
|
print(
|
|
f"point series={series} replicas={replicas} api_cpu={api_cpu} "
|
|
f"api_mem={api_mem} db_cpu={db_cpu} http_workers={HTTP_WORKERS} "
|
|
f"db_pool={DB_POOL_SIZE} nginx_workers={NGINX_WORKERS}",
|
|
file=sys.stderr,
|
|
)
|
|
down = compose(["down"], env)
|
|
if down.returncode != 0:
|
|
raise SystemExit(down.returncode)
|
|
up = compose(
|
|
["up", "-d", "--build", "--scale", f"api={replicas}"],
|
|
env,
|
|
)
|
|
if up.returncode != 0:
|
|
raise SystemExit(up.returncode)
|
|
wait_stack(env, replicas)
|
|
limits = verify_limits(env, replicas, api_cpu, api_mem, db_cpu)
|
|
print(
|
|
f"inspect ok api_sum_memory={limits['api_memory_sum']} "
|
|
f"api_nano={limits['api'][0]['nano_cpus']} db_nano={limits['db']['nano_cpus']} "
|
|
f"nginx={limits['lb']['nginx_worker_processes']}",
|
|
file=sys.stderr,
|
|
)
|
|
|
|
warmup = subprocess.run(
|
|
[
|
|
sys.executable,
|
|
str(ROOT / "tests" / "load.py"),
|
|
"--duration",
|
|
"10",
|
|
"--concurrency",
|
|
"50",
|
|
"--output",
|
|
f"/tmp/{label}-warmup.json",
|
|
],
|
|
cwd=ROOT,
|
|
text=True,
|
|
capture_output=True,
|
|
)
|
|
if warmup.returncode != 0:
|
|
sys.stderr.write(warmup.stdout)
|
|
sys.stderr.write(warmup.stderr)
|
|
raise SystemExit("warmup failed")
|
|
|
|
samples = []
|
|
stop = threading.Event()
|
|
thread = threading.Thread(target=sample_stats, args=(stop, samples))
|
|
thread.start()
|
|
try:
|
|
measured = subprocess.run(
|
|
[
|
|
sys.executable,
|
|
str(ROOT / "tests" / "load.py"),
|
|
"--duration",
|
|
"30",
|
|
"--concurrency",
|
|
"50",
|
|
"--output",
|
|
str(raw),
|
|
],
|
|
cwd=ROOT,
|
|
text=True,
|
|
capture_output=True,
|
|
)
|
|
finally:
|
|
stop.set()
|
|
thread.join()
|
|
if measured.returncode != 0:
|
|
sys.stderr.write(measured.stdout)
|
|
sys.stderr.write(measured.stderr)
|
|
raise SystemExit("measured run failed")
|
|
load = json.loads(raw.read_text())
|
|
report = {
|
|
"series": series,
|
|
"api_replicas": replicas,
|
|
"api_cpu_limit": float(api_cpu),
|
|
"api_mem_limit": api_mem,
|
|
"api_mem_limit_bytes": mem_bytes(api_mem),
|
|
"api_mem_total_bytes": TOTAL_API_RAM,
|
|
"db_cpu_limit": float(db_cpu),
|
|
"http_workers": int(HTTP_WORKERS),
|
|
"pg_pool": int(DB_POOL_SIZE),
|
|
"nginx_worker_processes": NGINX_WORKERS,
|
|
"lb_cpu_limit": float(LB_CPU_LIMIT),
|
|
"warmup_seconds": 10,
|
|
"inspect": limits,
|
|
"cpu": summarize_cpu(samples),
|
|
"load": load,
|
|
}
|
|
dest.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n")
|
|
lat = load["successful_latency_ms"]
|
|
summary = {
|
|
"file": str(dest.relative_to(ROOT)),
|
|
"rps": load["successful_rps"],
|
|
"p50": lat["p50"],
|
|
"p95": lat["p95"],
|
|
"p99": lat["p99"],
|
|
"errors": load["failed_requests"],
|
|
"error_ratio": load["error_ratio"],
|
|
"api_mem_total_bytes": TOTAL_API_RAM,
|
|
"cpu": report["cpu"],
|
|
}
|
|
print(json.dumps(summary, ensure_ascii=False))
|
|
return summary
|
|
|
|
|
|
def main():
|
|
if len(sys.argv) != 1:
|
|
raise SystemExit("usage: python3 scripts/run_track1.py")
|
|
for series, replicas, api_cpu, api_mem, db_cpu in series_points():
|
|
run_point(series, replicas, api_cpu, api_mem, db_cpu)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|