forked from mtusi2026/mtusi2026lab3
fix reproducibility
This commit is contained in:
+5
-15
@@ -55,8 +55,6 @@ def dual_axis(ax, xs, rps, p99, xlabel, title):
|
||||
def main():
|
||||
series_a = sorted(load_series("A"), key=lambda row: row["replicas"])
|
||||
series_b = sorted(load_series("B"), key=lambda row: row["db_cpu"])
|
||||
repeat = load_series("R")
|
||||
tuned = load_series("P")
|
||||
if not series_a or not series_b:
|
||||
raise SystemExit("need series A and B JSON in evidence/")
|
||||
|
||||
@@ -66,30 +64,22 @@ def main():
|
||||
[row["replicas"] for row in series_a],
|
||||
[row["rps"] for row in series_a],
|
||||
[row["p99"] for row in series_a],
|
||||
"API replicas (total CPU = 1.0, DB CPU = 2.0)",
|
||||
"Серия A: реплики API при фиксированном суммарном CPU",
|
||||
"API replicas (total CPU = 1.0, total RAM = 512 MiB, DB CPU = 2.0)",
|
||||
"Серия A: реплики API при фиксированных суммарных CPU и RAM",
|
||||
)
|
||||
fig.tight_layout()
|
||||
fig.savefig(EVIDENCE / "api_scaling.png", dpi=140)
|
||||
plt.close(fig)
|
||||
|
||||
fig, ax = plt.subplots(figsize=(7.2, 4.4))
|
||||
twin = dual_axis(
|
||||
dual_axis(
|
||||
ax,
|
||||
[row["db_cpu"] for row in series_b],
|
||||
[row["rps"] for row in series_b],
|
||||
[row["p99"] for row in series_b],
|
||||
"PostgreSQL CPU limit (4 API × 0.25 CPU)",
|
||||
"Серия B: CPU базы и парный прогон тюнинга",
|
||||
"PostgreSQL CPU limit (4 API × 0.25 CPU, total API RAM = 512 MiB)",
|
||||
"Серия B: RPS и p99 от CPU базы",
|
||||
)
|
||||
if repeat and tuned:
|
||||
base, point = repeat[-1], tuned[-1]
|
||||
ax.scatter([base["db_cpu"]], [base["rps"]], marker="x", s=80, color="#1f4e79", zorder=5, label="repeat baseline RPS")
|
||||
ax.scatter([point["db_cpu"] + 0.08], [point["rps"]], marker="*", s=160, color="#1f4e79", zorder=5, label="tuned RPS")
|
||||
twin.scatter([base["db_cpu"]], [base["p99"]], marker="x", s=80, color="#b85c38", zorder=5, label="repeat baseline p99")
|
||||
twin.scatter([point["db_cpu"] + 0.08], [point["p99"]], marker="*", s=160, color="#b85c38", zorder=5, label="tuned p99")
|
||||
handles = ax.get_lines() + twin.get_lines() + list(ax.collections) + list(twin.collections)
|
||||
ax.legend(handles, [h.get_label() for h in handles], loc="best", fontsize=8)
|
||||
fig.tight_layout()
|
||||
fig.savefig(EVIDENCE / "db_scaling.png", dpi=140)
|
||||
plt.close(fig)
|
||||
|
||||
+207
-67
@@ -1,5 +1,9 @@
|
||||
#!/usr/bin/env python3
|
||||
"""One measured load.py run under explicit Compose CPU limits. Prints nothing secret."""
|
||||
"""Series A and B with a fixed total API RAM budget and the base 8/8/1 profile.
|
||||
|
||||
The profile is applied through Compose environment, then checked with docker inspect
|
||||
before the load window. Nothing secret is printed.
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
@@ -8,30 +12,66 @@ import threading
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from decimal import Decimal
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
HTTP_WORKERS = "8"
|
||||
DB_POOL_SIZE = "8"
|
||||
NGINX_WORKERS = "1"
|
||||
LB_CPU_LIMIT = "0.25"
|
||||
TOTAL_API_RAM = 512 * 1024 * 1024
|
||||
|
||||
|
||||
def env_for(api_replicas, api_cpu, db_cpu):
|
||||
def mem_bytes(spec):
|
||||
text = str(spec).strip()
|
||||
units = {"b": 1, "k": 1024, "m": 1024**2, "g": 1024**3}
|
||||
unit = text[-1].lower()
|
||||
if unit not in units:
|
||||
raise SystemExit(f"bad memory spec: {spec}")
|
||||
return int(text[:-1]) * units[unit]
|
||||
|
||||
|
||||
def cpu_nanos(spec):
|
||||
return int(Decimal(str(spec)) * Decimal(1_000_000_000))
|
||||
|
||||
|
||||
def series_points():
|
||||
points = []
|
||||
for replicas, api_cpu in ((1, "1.0"), (2, "0.5"), (4, "0.25")):
|
||||
per = TOTAL_API_RAM // replicas // (1024 * 1024)
|
||||
points.append(("A", replicas, api_cpu, f"{per}m", "2.0"))
|
||||
for db_cpu in ("1.0", "2.0", "4.0"):
|
||||
per = TOTAL_API_RAM // 4 // (1024 * 1024)
|
||||
points.append(("B", 4, "0.25", f"{per}m", db_cpu))
|
||||
return points
|
||||
|
||||
|
||||
def env_for(replicas, api_cpu, api_mem, db_cpu):
|
||||
env = os.environ.copy()
|
||||
env["API_REPLICAS"] = str(api_replicas)
|
||||
env["API_REPLICAS"] = str(replicas)
|
||||
env["API_CPU_LIMIT"] = str(api_cpu)
|
||||
env["API_MEM_LIMIT"] = api_mem
|
||||
env["DB_CPU_LIMIT"] = str(db_cpu)
|
||||
env["HTTP_WORKERS"] = HTTP_WORKERS
|
||||
env["DB_POOL_SIZE"] = DB_POOL_SIZE
|
||||
env["NGINX_WORKERS"] = NGINX_WORKERS
|
||||
env["LB_CPU_LIMIT"] = LB_CPU_LIMIT
|
||||
return env
|
||||
|
||||
|
||||
def compose(args, env):
|
||||
def compose(args, env, capture=False):
|
||||
result = subprocess.run(
|
||||
["docker", "compose", "--env-file", str(ROOT / ".env"), *args],
|
||||
cwd=ROOT,
|
||||
env=env,
|
||||
check=False,
|
||||
text=True,
|
||||
capture_output=True,
|
||||
capture_output=capture,
|
||||
)
|
||||
if result.returncode != 0:
|
||||
sys.stderr.write(result.stderr or result.stdout)
|
||||
if result.returncode != 0 and capture:
|
||||
sys.stderr.write(result.stderr or "")
|
||||
sys.stderr.write(result.stdout or "")
|
||||
return result
|
||||
|
||||
|
||||
@@ -39,7 +79,7 @@ def wait_stack(env, replicas, timeout=180):
|
||||
deadline = time.monotonic() + timeout
|
||||
last = ""
|
||||
while time.monotonic() < deadline:
|
||||
ps = compose(["ps", "api", "--format", "{{.Health}}"], env)
|
||||
ps = compose(["ps", "api", "--format", "{{.Health}}"], env, capture=True)
|
||||
health = [line.strip() for line in ps.stdout.splitlines() if line.strip()]
|
||||
ready = False
|
||||
try:
|
||||
@@ -54,18 +94,118 @@ def wait_stack(env, replicas, timeout=180):
|
||||
raise SystemExit("stack did not become ready: " + last)
|
||||
|
||||
|
||||
def observed_nanos(env):
|
||||
ids = compose(["ps", "-q"], env).stdout.split()
|
||||
found = []
|
||||
for cid in ids:
|
||||
info = subprocess.check_output(
|
||||
["docker", "inspect", "-f", "{{.Name}} {{.HostConfig.NanoCpus}} {{.Config.Labels}}", cid],
|
||||
text=True,
|
||||
).strip()
|
||||
found.append(info)
|
||||
def container_ids(env, service):
|
||||
result = compose(["ps", "-q", service], env, capture=True)
|
||||
if result.returncode != 0:
|
||||
raise SystemExit(f"compose ps {service} failed")
|
||||
return [line for line in result.stdout.split() if line]
|
||||
|
||||
|
||||
def inspect_container(cid):
|
||||
raw = subprocess.check_output(["docker", "inspect", cid], text=True)
|
||||
return json.loads(raw)[0]
|
||||
|
||||
|
||||
def env_map(info):
|
||||
found = {}
|
||||
for item in info["Config"].get("Env") or []:
|
||||
key, _, value = item.partition("=")
|
||||
found[key] = value
|
||||
return found
|
||||
|
||||
|
||||
def verify_limits(env, replicas, api_cpu, api_mem, db_cpu):
|
||||
expect_api_nanos = cpu_nanos(api_cpu)
|
||||
expect_api_mem = mem_bytes(api_mem)
|
||||
expect_db_nanos = cpu_nanos(db_cpu)
|
||||
expect_lb_nanos = cpu_nanos(LB_CPU_LIMIT)
|
||||
api_ids = container_ids(env, "api")
|
||||
if len(api_ids) != replicas:
|
||||
raise SystemExit(f"expected {replicas} api containers, found {len(api_ids)}")
|
||||
|
||||
api_rows = []
|
||||
for cid in api_ids:
|
||||
info = inspect_container(cid)
|
||||
nanos = info["HostConfig"]["NanoCpus"]
|
||||
memory = info["HostConfig"]["Memory"]
|
||||
present = env_map(info)
|
||||
name = info["Name"].lstrip("/")
|
||||
if nanos != expect_api_nanos or memory != expect_api_mem:
|
||||
raise SystemExit(
|
||||
f"{name}: NanoCpus={nanos} Memory={memory}, "
|
||||
f"expected {expect_api_nanos} and {expect_api_mem}"
|
||||
)
|
||||
if present.get("HTTP_WORKERS") != HTTP_WORKERS or present.get("DB_POOL_SIZE") != DB_POOL_SIZE:
|
||||
raise SystemExit(
|
||||
f"{name}: HTTP_WORKERS={present.get('HTTP_WORKERS')} "
|
||||
f"DB_POOL_SIZE={present.get('DB_POOL_SIZE')}"
|
||||
)
|
||||
api_rows.append(
|
||||
{
|
||||
"name": name,
|
||||
"nano_cpus": nanos,
|
||||
"memory": memory,
|
||||
"http_workers": present["HTTP_WORKERS"],
|
||||
"db_pool_size": present["DB_POOL_SIZE"],
|
||||
}
|
||||
)
|
||||
total_mem = sum(row["memory"] for row in api_rows)
|
||||
if total_mem != TOTAL_API_RAM:
|
||||
raise SystemExit(f"api RAM sum {total_mem} != {TOTAL_API_RAM}")
|
||||
|
||||
db_ids = container_ids(env, "db")
|
||||
if len(db_ids) != 1:
|
||||
raise SystemExit(f"expected 1 db container, found {len(db_ids)}")
|
||||
db_info = inspect_container(db_ids[0])
|
||||
db_nanos = db_info["HostConfig"]["NanoCpus"]
|
||||
if db_nanos != expect_db_nanos:
|
||||
raise SystemExit(f"db NanoCpus={db_nanos}, expected {expect_db_nanos}")
|
||||
|
||||
lb_ids = container_ids(env, "lb")
|
||||
if len(lb_ids) != 1:
|
||||
raise SystemExit(f"expected 1 lb container, found {len(lb_ids)}")
|
||||
lb_info = inspect_container(lb_ids[0])
|
||||
lb_nanos = lb_info["HostConfig"]["NanoCpus"]
|
||||
lb_env = env_map(lb_info)
|
||||
if lb_nanos != expect_lb_nanos or lb_env.get("NGINX_WORKERS") != NGINX_WORKERS:
|
||||
raise SystemExit(
|
||||
f"lb NanoCpus={lb_nanos} NGINX_WORKERS={lb_env.get('NGINX_WORKERS')}"
|
||||
)
|
||||
rendered = compose(["exec", "-T", "lb", "nginx", "-T"], env, capture=True)
|
||||
if rendered.returncode != 0:
|
||||
raise SystemExit("nginx -T failed")
|
||||
directive = ""
|
||||
rendered_text = (rendered.stdout or "") + "\n" + (rendered.stderr or "")
|
||||
for line in rendered_text.splitlines():
|
||||
stripped = line.strip()
|
||||
if stripped.startswith("worker_processes"):
|
||||
directive = stripped
|
||||
break
|
||||
expected_directive = f"worker_processes {NGINX_WORKERS};"
|
||||
if directive != expected_directive:
|
||||
raise SystemExit(f"nginx config has {directive!r}, expected {expected_directive!r}")
|
||||
|
||||
logs = compose(["logs", "api"], env, capture=True)
|
||||
if logs.returncode != 0:
|
||||
raise SystemExit("api logs failed")
|
||||
pool_line = f"pg pool ready ({DB_POOL_SIZE} connections)"
|
||||
worker_line = f"http workers: {HTTP_WORKERS}"
|
||||
if logs.stdout.count(pool_line) < replicas or logs.stdout.count(worker_line) < replicas:
|
||||
raise SystemExit("api logs do not show the base pool and worker profile on every replica")
|
||||
|
||||
return {
|
||||
"verified": True,
|
||||
"api": api_rows,
|
||||
"api_memory_sum": total_mem,
|
||||
"db": {"name": db_info["Name"].lstrip("/"), "nano_cpus": db_nanos},
|
||||
"lb": {
|
||||
"name": lb_info["Name"].lstrip("/"),
|
||||
"nano_cpus": lb_nanos,
|
||||
"nginx_worker_processes": directive,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def sample_stats(stop, bucket):
|
||||
while not stop.is_set():
|
||||
proc = subprocess.run(
|
||||
@@ -81,9 +221,8 @@ def sample_stats(stop, bucket):
|
||||
if len(parts) != 3:
|
||||
continue
|
||||
name, cpu, mem = parts
|
||||
cpu_val = cpu.strip().rstrip("%")
|
||||
try:
|
||||
cpu_num = float(cpu_val)
|
||||
cpu_num = float(cpu.strip().rstrip("%"))
|
||||
except ValueError:
|
||||
continue
|
||||
rows.append({"name": name, "cpu_percent": cpu_num, "mem": mem.strip()})
|
||||
@@ -113,43 +252,34 @@ def summarize_cpu(samples):
|
||||
return out
|
||||
|
||||
|
||||
def main():
|
||||
if len(sys.argv) != 7:
|
||||
raise SystemExit(
|
||||
"usage: run_track1.py SERIES REPLICAS API_CPU DB_CPU POOL NGINX_WORKERS"
|
||||
)
|
||||
series, replicas, api_cpu, db_cpu, pool, nginx_workers = sys.argv[1:]
|
||||
replicas = int(replicas)
|
||||
env = env_for(replicas, api_cpu, db_cpu)
|
||||
def run_point(series, replicas, api_cpu, api_mem, db_cpu):
|
||||
env = env_for(replicas, api_cpu, api_mem, db_cpu)
|
||||
label = f"track1-{series}-r{replicas}-apicpu{api_cpu}-dbcpu{db_cpu}"
|
||||
raw = ROOT / "evidence" / f"{label}.load.json"
|
||||
dest = ROOT / "evidence" / f"{label}.json"
|
||||
|
||||
up_db = None
|
||||
for _ in range(3):
|
||||
up_db = compose(
|
||||
["up", "-d", "--no-deps", "--force-recreate", "db"],
|
||||
env,
|
||||
)
|
||||
if up_db.returncode == 0:
|
||||
break
|
||||
time.sleep(2)
|
||||
if up_db.returncode != 0:
|
||||
sys.stderr.write(up_db.stderr)
|
||||
raise SystemExit(up_db.returncode)
|
||||
up_api = compose(
|
||||
["up", "-d", "--no-deps", "--force-recreate", "--scale", f"api={replicas}", "api"],
|
||||
print(
|
||||
f"point series={series} replicas={replicas} api_cpu={api_cpu} "
|
||||
f"api_mem={api_mem} db_cpu={db_cpu} http_workers={HTTP_WORKERS} "
|
||||
f"db_pool={DB_POOL_SIZE} nginx_workers={NGINX_WORKERS}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
down = compose(["down"], env)
|
||||
if down.returncode != 0:
|
||||
raise SystemExit(down.returncode)
|
||||
up = compose(
|
||||
["up", "-d", "--build", "--scale", f"api={replicas}"],
|
||||
env,
|
||||
)
|
||||
if up_api.returncode != 0:
|
||||
sys.stderr.write(up_api.stderr)
|
||||
raise SystemExit(up_api.returncode)
|
||||
up_lb = compose(["up", "-d", "--no-deps", "lb"], env)
|
||||
if up_lb.returncode != 0:
|
||||
sys.stderr.write(up_lb.stderr)
|
||||
raise SystemExit(up_lb.returncode)
|
||||
if up.returncode != 0:
|
||||
raise SystemExit(up.returncode)
|
||||
wait_stack(env, replicas)
|
||||
limits = observed_nanos(env)
|
||||
limits = verify_limits(env, replicas, api_cpu, api_mem, db_cpu)
|
||||
print(
|
||||
f"inspect ok api_sum_memory={limits['api_memory_sum']} "
|
||||
f"api_nano={limits['api'][0]['nano_cpus']} db_nano={limits['db']['nano_cpus']} "
|
||||
f"nginx={limits['lb']['nginx_worker_processes']}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
|
||||
warmup = subprocess.run(
|
||||
[
|
||||
@@ -203,9 +333,14 @@ def main():
|
||||
"series": series,
|
||||
"api_replicas": replicas,
|
||||
"api_cpu_limit": float(api_cpu),
|
||||
"api_mem_limit": api_mem,
|
||||
"api_mem_limit_bytes": mem_bytes(api_mem),
|
||||
"api_mem_total_bytes": TOTAL_API_RAM,
|
||||
"db_cpu_limit": float(db_cpu),
|
||||
"pg_pool": int(pool),
|
||||
"nginx_worker_processes": nginx_workers,
|
||||
"http_workers": int(HTTP_WORKERS),
|
||||
"pg_pool": int(DB_POOL_SIZE),
|
||||
"nginx_worker_processes": NGINX_WORKERS,
|
||||
"lb_cpu_limit": float(LB_CPU_LIMIT),
|
||||
"warmup_seconds": 10,
|
||||
"inspect": limits,
|
||||
"cpu": summarize_cpu(samples),
|
||||
@@ -213,21 +348,26 @@ def main():
|
||||
}
|
||||
dest.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n")
|
||||
lat = load["successful_latency_ms"]
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"file": str(dest.relative_to(ROOT)),
|
||||
"rps": load["successful_rps"],
|
||||
"p50": lat["p50"],
|
||||
"p95": lat["p95"],
|
||||
"p99": lat["p99"],
|
||||
"errors": load["failed_requests"],
|
||||
"error_ratio": load["error_ratio"],
|
||||
"cpu": report["cpu"],
|
||||
},
|
||||
ensure_ascii=False,
|
||||
)
|
||||
)
|
||||
summary = {
|
||||
"file": str(dest.relative_to(ROOT)),
|
||||
"rps": load["successful_rps"],
|
||||
"p50": lat["p50"],
|
||||
"p95": lat["p95"],
|
||||
"p99": lat["p99"],
|
||||
"errors": load["failed_requests"],
|
||||
"error_ratio": load["error_ratio"],
|
||||
"api_mem_total_bytes": TOTAL_API_RAM,
|
||||
"cpu": report["cpu"],
|
||||
}
|
||||
print(json.dumps(summary, ensure_ascii=False))
|
||||
return summary
|
||||
|
||||
|
||||
def main():
|
||||
if len(sys.argv) != 1:
|
||||
raise SystemExit("usage: python3 scripts/run_track1.py")
|
||||
for series, replicas, api_cpu, api_mem, db_cpu in series_points():
|
||||
run_point(series, replicas, api_cpu, api_mem, db_cpu)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
Reference in New Issue
Block a user