forked from mtusi2026/mtusi2026lab3
235 lines
7.0 KiB
Python
Executable File
235 lines
7.0 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""One measured load.py run under explicit Compose CPU limits. Prints nothing secret."""
|
|
import json
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
import threading
|
|
import time
|
|
import urllib.error
|
|
import urllib.request
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
|
|
|
|
def env_for(api_replicas, api_cpu, db_cpu):
|
|
env = os.environ.copy()
|
|
env["API_REPLICAS"] = str(api_replicas)
|
|
env["API_CPU_LIMIT"] = str(api_cpu)
|
|
env["DB_CPU_LIMIT"] = str(db_cpu)
|
|
return env
|
|
|
|
|
|
def compose(args, env):
|
|
result = subprocess.run(
|
|
["docker", "compose", "--env-file", str(ROOT / ".env"), *args],
|
|
cwd=ROOT,
|
|
env=env,
|
|
check=False,
|
|
text=True,
|
|
capture_output=True,
|
|
)
|
|
if result.returncode != 0:
|
|
sys.stderr.write(result.stderr or result.stdout)
|
|
return result
|
|
|
|
|
|
def wait_stack(env, replicas, timeout=180):
|
|
deadline = time.monotonic() + timeout
|
|
last = ""
|
|
while time.monotonic() < deadline:
|
|
ps = compose(["ps", "api", "--format", "{{.Health}}"], env)
|
|
health = [line.strip() for line in ps.stdout.splitlines() if line.strip()]
|
|
ready = False
|
|
try:
|
|
with urllib.request.urlopen("http://127.0.0.1:8080/health/ready", timeout=2) as resp:
|
|
ready = resp.status == 200
|
|
except (urllib.error.URLError, TimeoutError, OSError):
|
|
ready = False
|
|
last = f"api_health={health} ready={ready}"
|
|
if len(health) == replicas and all(item == "healthy" for item in health) and ready:
|
|
return
|
|
time.sleep(2)
|
|
raise SystemExit("stack did not become ready: " + last)
|
|
|
|
|
|
def observed_nanos(env):
|
|
ids = compose(["ps", "-q"], env).stdout.split()
|
|
found = []
|
|
for cid in ids:
|
|
info = subprocess.check_output(
|
|
["docker", "inspect", "-f", "{{.Name}} {{.HostConfig.NanoCpus}} {{.Config.Labels}}", cid],
|
|
text=True,
|
|
).strip()
|
|
found.append(info)
|
|
return found
|
|
|
|
|
|
def sample_stats(stop, bucket):
|
|
while not stop.is_set():
|
|
proc = subprocess.run(
|
|
["docker", "stats", "--no-stream", "--format", "{{.Name}}\t{{.CPUPerc}}\t{{.MemUsage}}"],
|
|
text=True,
|
|
capture_output=True,
|
|
check=False,
|
|
)
|
|
stamp = time.time()
|
|
rows = []
|
|
for line in proc.stdout.splitlines():
|
|
parts = line.split("\t")
|
|
if len(parts) != 3:
|
|
continue
|
|
name, cpu, mem = parts
|
|
cpu_val = cpu.strip().rstrip("%")
|
|
try:
|
|
cpu_num = float(cpu_val)
|
|
except ValueError:
|
|
continue
|
|
rows.append({"name": name, "cpu_percent": cpu_num, "mem": mem.strip()})
|
|
if rows:
|
|
bucket.append({"t": stamp, "rows": rows})
|
|
stop.wait(3)
|
|
|
|
|
|
def summarize_cpu(samples):
|
|
acc = {}
|
|
for sample in samples:
|
|
for row in sample["rows"]:
|
|
name = row["name"].lstrip("/")
|
|
service = "other"
|
|
for prefix in ("api", "db", "cache", "lb"):
|
|
if f"-{prefix}-" in name or name.endswith(prefix):
|
|
service = prefix
|
|
break
|
|
acc.setdefault(service, []).append(row["cpu_percent"])
|
|
out = {}
|
|
for service, values in acc.items():
|
|
out[service] = {
|
|
"samples": len(values),
|
|
"mean_cpu_percent": round(sum(values) / len(values), 2),
|
|
"max_cpu_percent": round(max(values), 2),
|
|
}
|
|
return out
|
|
|
|
|
|
def main():
|
|
if len(sys.argv) != 7:
|
|
raise SystemExit(
|
|
"usage: run_track1.py SERIES REPLICAS API_CPU DB_CPU POOL NGINX_WORKERS"
|
|
)
|
|
series, replicas, api_cpu, db_cpu, pool, nginx_workers = sys.argv[1:]
|
|
replicas = int(replicas)
|
|
env = env_for(replicas, api_cpu, db_cpu)
|
|
label = f"track1-{series}-r{replicas}-apicpu{api_cpu}-dbcpu{db_cpu}"
|
|
raw = ROOT / "evidence" / f"{label}.load.json"
|
|
dest = ROOT / "evidence" / f"{label}.json"
|
|
|
|
up_db = None
|
|
for _ in range(3):
|
|
up_db = compose(
|
|
["up", "-d", "--no-deps", "--force-recreate", "db"],
|
|
env,
|
|
)
|
|
if up_db.returncode == 0:
|
|
break
|
|
time.sleep(2)
|
|
if up_db.returncode != 0:
|
|
sys.stderr.write(up_db.stderr)
|
|
raise SystemExit(up_db.returncode)
|
|
up_api = compose(
|
|
["up", "-d", "--no-deps", "--force-recreate", "--scale", f"api={replicas}", "api"],
|
|
env,
|
|
)
|
|
if up_api.returncode != 0:
|
|
sys.stderr.write(up_api.stderr)
|
|
raise SystemExit(up_api.returncode)
|
|
up_lb = compose(["up", "-d", "--no-deps", "lb"], env)
|
|
if up_lb.returncode != 0:
|
|
sys.stderr.write(up_lb.stderr)
|
|
raise SystemExit(up_lb.returncode)
|
|
wait_stack(env, replicas)
|
|
limits = observed_nanos(env)
|
|
|
|
warmup = subprocess.run(
|
|
[
|
|
sys.executable,
|
|
str(ROOT / "tests" / "load.py"),
|
|
"--duration",
|
|
"10",
|
|
"--concurrency",
|
|
"50",
|
|
"--output",
|
|
f"/tmp/{label}-warmup.json",
|
|
],
|
|
cwd=ROOT,
|
|
text=True,
|
|
capture_output=True,
|
|
)
|
|
if warmup.returncode != 0:
|
|
sys.stderr.write(warmup.stdout)
|
|
sys.stderr.write(warmup.stderr)
|
|
raise SystemExit("warmup failed")
|
|
|
|
samples = []
|
|
stop = threading.Event()
|
|
thread = threading.Thread(target=sample_stats, args=(stop, samples))
|
|
thread.start()
|
|
try:
|
|
measured = subprocess.run(
|
|
[
|
|
sys.executable,
|
|
str(ROOT / "tests" / "load.py"),
|
|
"--duration",
|
|
"30",
|
|
"--concurrency",
|
|
"50",
|
|
"--output",
|
|
str(raw),
|
|
],
|
|
cwd=ROOT,
|
|
text=True,
|
|
capture_output=True,
|
|
)
|
|
finally:
|
|
stop.set()
|
|
thread.join()
|
|
if measured.returncode != 0:
|
|
sys.stderr.write(measured.stdout)
|
|
sys.stderr.write(measured.stderr)
|
|
raise SystemExit("measured run failed")
|
|
load = json.loads(raw.read_text())
|
|
report = {
|
|
"series": series,
|
|
"api_replicas": replicas,
|
|
"api_cpu_limit": float(api_cpu),
|
|
"db_cpu_limit": float(db_cpu),
|
|
"pg_pool": int(pool),
|
|
"nginx_worker_processes": nginx_workers,
|
|
"warmup_seconds": 10,
|
|
"inspect": limits,
|
|
"cpu": summarize_cpu(samples),
|
|
"load": load,
|
|
}
|
|
dest.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n")
|
|
lat = load["successful_latency_ms"]
|
|
print(
|
|
json.dumps(
|
|
{
|
|
"file": str(dest.relative_to(ROOT)),
|
|
"rps": load["successful_rps"],
|
|
"p50": lat["p50"],
|
|
"p95": lat["p95"],
|
|
"p99": lat["p99"],
|
|
"errors": load["failed_requests"],
|
|
"error_ratio": load["error_ratio"],
|
|
"cpu": report["cpu"],
|
|
},
|
|
ensure_ascii=False,
|
|
)
|
|
)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|