fix reproducibility

This commit is contained in:
2026-10-23 18:21:10 +03:00
parent a7f3ab11a7
commit 0ae9e07b09
21 changed files with 667 additions and 300 deletions
+5 -15
View File
@@ -55,8 +55,6 @@ def dual_axis(ax, xs, rps, p99, xlabel, title):
def main():
series_a = sorted(load_series("A"), key=lambda row: row["replicas"])
series_b = sorted(load_series("B"), key=lambda row: row["db_cpu"])
repeat = load_series("R")
tuned = load_series("P")
if not series_a or not series_b:
raise SystemExit("need series A and B JSON in evidence/")
@@ -66,30 +64,22 @@ def main():
[row["replicas"] for row in series_a],
[row["rps"] for row in series_a],
[row["p99"] for row in series_a],
"API replicas (total CPU = 1.0, DB CPU = 2.0)",
"Серия A: реплики API при фиксированном суммарном CPU",
"API replicas (total CPU = 1.0, total RAM = 512 MiB, DB CPU = 2.0)",
"Серия A: реплики API при фиксированных суммарных CPU и RAM",
)
fig.tight_layout()
fig.savefig(EVIDENCE / "api_scaling.png", dpi=140)
plt.close(fig)
fig, ax = plt.subplots(figsize=(7.2, 4.4))
twin = dual_axis(
dual_axis(
ax,
[row["db_cpu"] for row in series_b],
[row["rps"] for row in series_b],
[row["p99"] for row in series_b],
"PostgreSQL CPU limit (4 API × 0.25 CPU)",
"Серия B: CPU базы и парный прогон тюнинга",
"PostgreSQL CPU limit (4 API × 0.25 CPU, total API RAM = 512 MiB)",
"Серия B: RPS и p99 от CPU базы",
)
if repeat and tuned:
base, point = repeat[-1], tuned[-1]
ax.scatter([base["db_cpu"]], [base["rps"]], marker="x", s=80, color="#1f4e79", zorder=5, label="repeat baseline RPS")
ax.scatter([point["db_cpu"] + 0.08], [point["rps"]], marker="*", s=160, color="#1f4e79", zorder=5, label="tuned RPS")
twin.scatter([base["db_cpu"]], [base["p99"]], marker="x", s=80, color="#b85c38", zorder=5, label="repeat baseline p99")
twin.scatter([point["db_cpu"] + 0.08], [point["p99"]], marker="*", s=160, color="#b85c38", zorder=5, label="tuned p99")
handles = ax.get_lines() + twin.get_lines() + list(ax.collections) + list(twin.collections)
ax.legend(handles, [h.get_label() for h in handles], loc="best", fontsize=8)
fig.tight_layout()
fig.savefig(EVIDENCE / "db_scaling.png", dpi=140)
plt.close(fig)
+207 -67
View File
@@ -1,5 +1,9 @@
#!/usr/bin/env python3
"""One measured load.py run under explicit Compose CPU limits. Prints nothing secret."""
"""Series A and B with a fixed total API RAM budget and the base 8/8/1 profile.
The profile is applied through Compose environment, then checked with docker inspect
before the load window. Nothing secret is printed.
"""
import json
import os
import subprocess
@@ -8,30 +12,66 @@ import threading
import time
import urllib.error
import urllib.request
from decimal import Decimal
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
HTTP_WORKERS = "8"
DB_POOL_SIZE = "8"
NGINX_WORKERS = "1"
LB_CPU_LIMIT = "0.25"
TOTAL_API_RAM = 512 * 1024 * 1024
def env_for(api_replicas, api_cpu, db_cpu):
def mem_bytes(spec):
text = str(spec).strip()
units = {"b": 1, "k": 1024, "m": 1024**2, "g": 1024**3}
unit = text[-1].lower()
if unit not in units:
raise SystemExit(f"bad memory spec: {spec}")
return int(text[:-1]) * units[unit]
def cpu_nanos(spec):
return int(Decimal(str(spec)) * Decimal(1_000_000_000))
def series_points():
points = []
for replicas, api_cpu in ((1, "1.0"), (2, "0.5"), (4, "0.25")):
per = TOTAL_API_RAM // replicas // (1024 * 1024)
points.append(("A", replicas, api_cpu, f"{per}m", "2.0"))
for db_cpu in ("1.0", "2.0", "4.0"):
per = TOTAL_API_RAM // 4 // (1024 * 1024)
points.append(("B", 4, "0.25", f"{per}m", db_cpu))
return points
def env_for(replicas, api_cpu, api_mem, db_cpu):
env = os.environ.copy()
env["API_REPLICAS"] = str(api_replicas)
env["API_REPLICAS"] = str(replicas)
env["API_CPU_LIMIT"] = str(api_cpu)
env["API_MEM_LIMIT"] = api_mem
env["DB_CPU_LIMIT"] = str(db_cpu)
env["HTTP_WORKERS"] = HTTP_WORKERS
env["DB_POOL_SIZE"] = DB_POOL_SIZE
env["NGINX_WORKERS"] = NGINX_WORKERS
env["LB_CPU_LIMIT"] = LB_CPU_LIMIT
return env
def compose(args, env):
def compose(args, env, capture=False):
result = subprocess.run(
["docker", "compose", "--env-file", str(ROOT / ".env"), *args],
cwd=ROOT,
env=env,
check=False,
text=True,
capture_output=True,
capture_output=capture,
)
if result.returncode != 0:
sys.stderr.write(result.stderr or result.stdout)
if result.returncode != 0 and capture:
sys.stderr.write(result.stderr or "")
sys.stderr.write(result.stdout or "")
return result
@@ -39,7 +79,7 @@ def wait_stack(env, replicas, timeout=180):
deadline = time.monotonic() + timeout
last = ""
while time.monotonic() < deadline:
ps = compose(["ps", "api", "--format", "{{.Health}}"], env)
ps = compose(["ps", "api", "--format", "{{.Health}}"], env, capture=True)
health = [line.strip() for line in ps.stdout.splitlines() if line.strip()]
ready = False
try:
@@ -54,18 +94,118 @@ def wait_stack(env, replicas, timeout=180):
raise SystemExit("stack did not become ready: " + last)
def observed_nanos(env):
ids = compose(["ps", "-q"], env).stdout.split()
found = []
for cid in ids:
info = subprocess.check_output(
["docker", "inspect", "-f", "{{.Name}} {{.HostConfig.NanoCpus}} {{.Config.Labels}}", cid],
text=True,
).strip()
found.append(info)
def container_ids(env, service):
result = compose(["ps", "-q", service], env, capture=True)
if result.returncode != 0:
raise SystemExit(f"compose ps {service} failed")
return [line for line in result.stdout.split() if line]
def inspect_container(cid):
raw = subprocess.check_output(["docker", "inspect", cid], text=True)
return json.loads(raw)[0]
def env_map(info):
found = {}
for item in info["Config"].get("Env") or []:
key, _, value = item.partition("=")
found[key] = value
return found
def verify_limits(env, replicas, api_cpu, api_mem, db_cpu):
expect_api_nanos = cpu_nanos(api_cpu)
expect_api_mem = mem_bytes(api_mem)
expect_db_nanos = cpu_nanos(db_cpu)
expect_lb_nanos = cpu_nanos(LB_CPU_LIMIT)
api_ids = container_ids(env, "api")
if len(api_ids) != replicas:
raise SystemExit(f"expected {replicas} api containers, found {len(api_ids)}")
api_rows = []
for cid in api_ids:
info = inspect_container(cid)
nanos = info["HostConfig"]["NanoCpus"]
memory = info["HostConfig"]["Memory"]
present = env_map(info)
name = info["Name"].lstrip("/")
if nanos != expect_api_nanos or memory != expect_api_mem:
raise SystemExit(
f"{name}: NanoCpus={nanos} Memory={memory}, "
f"expected {expect_api_nanos} and {expect_api_mem}"
)
if present.get("HTTP_WORKERS") != HTTP_WORKERS or present.get("DB_POOL_SIZE") != DB_POOL_SIZE:
raise SystemExit(
f"{name}: HTTP_WORKERS={present.get('HTTP_WORKERS')} "
f"DB_POOL_SIZE={present.get('DB_POOL_SIZE')}"
)
api_rows.append(
{
"name": name,
"nano_cpus": nanos,
"memory": memory,
"http_workers": present["HTTP_WORKERS"],
"db_pool_size": present["DB_POOL_SIZE"],
}
)
total_mem = sum(row["memory"] for row in api_rows)
if total_mem != TOTAL_API_RAM:
raise SystemExit(f"api RAM sum {total_mem} != {TOTAL_API_RAM}")
db_ids = container_ids(env, "db")
if len(db_ids) != 1:
raise SystemExit(f"expected 1 db container, found {len(db_ids)}")
db_info = inspect_container(db_ids[0])
db_nanos = db_info["HostConfig"]["NanoCpus"]
if db_nanos != expect_db_nanos:
raise SystemExit(f"db NanoCpus={db_nanos}, expected {expect_db_nanos}")
lb_ids = container_ids(env, "lb")
if len(lb_ids) != 1:
raise SystemExit(f"expected 1 lb container, found {len(lb_ids)}")
lb_info = inspect_container(lb_ids[0])
lb_nanos = lb_info["HostConfig"]["NanoCpus"]
lb_env = env_map(lb_info)
if lb_nanos != expect_lb_nanos or lb_env.get("NGINX_WORKERS") != NGINX_WORKERS:
raise SystemExit(
f"lb NanoCpus={lb_nanos} NGINX_WORKERS={lb_env.get('NGINX_WORKERS')}"
)
rendered = compose(["exec", "-T", "lb", "nginx", "-T"], env, capture=True)
if rendered.returncode != 0:
raise SystemExit("nginx -T failed")
directive = ""
rendered_text = (rendered.stdout or "") + "\n" + (rendered.stderr or "")
for line in rendered_text.splitlines():
stripped = line.strip()
if stripped.startswith("worker_processes"):
directive = stripped
break
expected_directive = f"worker_processes {NGINX_WORKERS};"
if directive != expected_directive:
raise SystemExit(f"nginx config has {directive!r}, expected {expected_directive!r}")
logs = compose(["logs", "api"], env, capture=True)
if logs.returncode != 0:
raise SystemExit("api logs failed")
pool_line = f"pg pool ready ({DB_POOL_SIZE} connections)"
worker_line = f"http workers: {HTTP_WORKERS}"
if logs.stdout.count(pool_line) < replicas or logs.stdout.count(worker_line) < replicas:
raise SystemExit("api logs do not show the base pool and worker profile on every replica")
return {
"verified": True,
"api": api_rows,
"api_memory_sum": total_mem,
"db": {"name": db_info["Name"].lstrip("/"), "nano_cpus": db_nanos},
"lb": {
"name": lb_info["Name"].lstrip("/"),
"nano_cpus": lb_nanos,
"nginx_worker_processes": directive,
},
}
def sample_stats(stop, bucket):
while not stop.is_set():
proc = subprocess.run(
@@ -81,9 +221,8 @@ def sample_stats(stop, bucket):
if len(parts) != 3:
continue
name, cpu, mem = parts
cpu_val = cpu.strip().rstrip("%")
try:
cpu_num = float(cpu_val)
cpu_num = float(cpu.strip().rstrip("%"))
except ValueError:
continue
rows.append({"name": name, "cpu_percent": cpu_num, "mem": mem.strip()})
@@ -113,43 +252,34 @@ def summarize_cpu(samples):
return out
def main():
if len(sys.argv) != 7:
raise SystemExit(
"usage: run_track1.py SERIES REPLICAS API_CPU DB_CPU POOL NGINX_WORKERS"
)
series, replicas, api_cpu, db_cpu, pool, nginx_workers = sys.argv[1:]
replicas = int(replicas)
env = env_for(replicas, api_cpu, db_cpu)
def run_point(series, replicas, api_cpu, api_mem, db_cpu):
env = env_for(replicas, api_cpu, api_mem, db_cpu)
label = f"track1-{series}-r{replicas}-apicpu{api_cpu}-dbcpu{db_cpu}"
raw = ROOT / "evidence" / f"{label}.load.json"
dest = ROOT / "evidence" / f"{label}.json"
up_db = None
for _ in range(3):
up_db = compose(
["up", "-d", "--no-deps", "--force-recreate", "db"],
env,
)
if up_db.returncode == 0:
break
time.sleep(2)
if up_db.returncode != 0:
sys.stderr.write(up_db.stderr)
raise SystemExit(up_db.returncode)
up_api = compose(
["up", "-d", "--no-deps", "--force-recreate", "--scale", f"api={replicas}", "api"],
print(
f"point series={series} replicas={replicas} api_cpu={api_cpu} "
f"api_mem={api_mem} db_cpu={db_cpu} http_workers={HTTP_WORKERS} "
f"db_pool={DB_POOL_SIZE} nginx_workers={NGINX_WORKERS}",
file=sys.stderr,
)
down = compose(["down"], env)
if down.returncode != 0:
raise SystemExit(down.returncode)
up = compose(
["up", "-d", "--build", "--scale", f"api={replicas}"],
env,
)
if up_api.returncode != 0:
sys.stderr.write(up_api.stderr)
raise SystemExit(up_api.returncode)
up_lb = compose(["up", "-d", "--no-deps", "lb"], env)
if up_lb.returncode != 0:
sys.stderr.write(up_lb.stderr)
raise SystemExit(up_lb.returncode)
if up.returncode != 0:
raise SystemExit(up.returncode)
wait_stack(env, replicas)
limits = observed_nanos(env)
limits = verify_limits(env, replicas, api_cpu, api_mem, db_cpu)
print(
f"inspect ok api_sum_memory={limits['api_memory_sum']} "
f"api_nano={limits['api'][0]['nano_cpus']} db_nano={limits['db']['nano_cpus']} "
f"nginx={limits['lb']['nginx_worker_processes']}",
file=sys.stderr,
)
warmup = subprocess.run(
[
@@ -203,9 +333,14 @@ def main():
"series": series,
"api_replicas": replicas,
"api_cpu_limit": float(api_cpu),
"api_mem_limit": api_mem,
"api_mem_limit_bytes": mem_bytes(api_mem),
"api_mem_total_bytes": TOTAL_API_RAM,
"db_cpu_limit": float(db_cpu),
"pg_pool": int(pool),
"nginx_worker_processes": nginx_workers,
"http_workers": int(HTTP_WORKERS),
"pg_pool": int(DB_POOL_SIZE),
"nginx_worker_processes": NGINX_WORKERS,
"lb_cpu_limit": float(LB_CPU_LIMIT),
"warmup_seconds": 10,
"inspect": limits,
"cpu": summarize_cpu(samples),
@@ -213,21 +348,26 @@ def main():
}
dest.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n")
lat = load["successful_latency_ms"]
print(
json.dumps(
{
"file": str(dest.relative_to(ROOT)),
"rps": load["successful_rps"],
"p50": lat["p50"],
"p95": lat["p95"],
"p99": lat["p99"],
"errors": load["failed_requests"],
"error_ratio": load["error_ratio"],
"cpu": report["cpu"],
},
ensure_ascii=False,
)
)
summary = {
"file": str(dest.relative_to(ROOT)),
"rps": load["successful_rps"],
"p50": lat["p50"],
"p95": lat["p95"],
"p99": lat["p99"],
"errors": load["failed_requests"],
"error_ratio": load["error_ratio"],
"api_mem_total_bytes": TOTAL_API_RAM,
"cpu": report["cpu"],
}
print(json.dumps(summary, ensure_ascii=False))
return summary
def main():
if len(sys.argv) != 1:
raise SystemExit("usage: python3 scripts/run_track1.py")
for series, replicas, api_cpu, api_mem, db_cpu in series_points():
run_point(series, replicas, api_cpu, api_mem, db_cpu)
if __name__ == "__main__":