Files
mtusi2026lab3/scripts/run_track1.py
T
2026-10-23 18:21:10 +03:00

375 lines
12 KiB
Python
Executable File

#!/usr/bin/env python3
"""Series A and B with a fixed total API RAM budget and the base 8/8/1 profile.
The profile is applied through Compose environment, then checked with docker inspect
before the load window. Nothing secret is printed.
"""
import json
import os
import subprocess
import sys
import threading
import time
import urllib.error
import urllib.request
from decimal import Decimal
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
HTTP_WORKERS = "8"
DB_POOL_SIZE = "8"
NGINX_WORKERS = "1"
LB_CPU_LIMIT = "0.25"
TOTAL_API_RAM = 512 * 1024 * 1024
def mem_bytes(spec):
text = str(spec).strip()
units = {"b": 1, "k": 1024, "m": 1024**2, "g": 1024**3}
unit = text[-1].lower()
if unit not in units:
raise SystemExit(f"bad memory spec: {spec}")
return int(text[:-1]) * units[unit]
def cpu_nanos(spec):
return int(Decimal(str(spec)) * Decimal(1_000_000_000))
def series_points():
points = []
for replicas, api_cpu in ((1, "1.0"), (2, "0.5"), (4, "0.25")):
per = TOTAL_API_RAM // replicas // (1024 * 1024)
points.append(("A", replicas, api_cpu, f"{per}m", "2.0"))
for db_cpu in ("1.0", "2.0", "4.0"):
per = TOTAL_API_RAM // 4 // (1024 * 1024)
points.append(("B", 4, "0.25", f"{per}m", db_cpu))
return points
def env_for(replicas, api_cpu, api_mem, db_cpu):
env = os.environ.copy()
env["API_REPLICAS"] = str(replicas)
env["API_CPU_LIMIT"] = str(api_cpu)
env["API_MEM_LIMIT"] = api_mem
env["DB_CPU_LIMIT"] = str(db_cpu)
env["HTTP_WORKERS"] = HTTP_WORKERS
env["DB_POOL_SIZE"] = DB_POOL_SIZE
env["NGINX_WORKERS"] = NGINX_WORKERS
env["LB_CPU_LIMIT"] = LB_CPU_LIMIT
return env
def compose(args, env, capture=False):
result = subprocess.run(
["docker", "compose", "--env-file", str(ROOT / ".env"), *args],
cwd=ROOT,
env=env,
check=False,
text=True,
capture_output=capture,
)
if result.returncode != 0 and capture:
sys.stderr.write(result.stderr or "")
sys.stderr.write(result.stdout or "")
return result
def wait_stack(env, replicas, timeout=180):
deadline = time.monotonic() + timeout
last = ""
while time.monotonic() < deadline:
ps = compose(["ps", "api", "--format", "{{.Health}}"], env, capture=True)
health = [line.strip() for line in ps.stdout.splitlines() if line.strip()]
ready = False
try:
with urllib.request.urlopen("http://127.0.0.1:8080/health/ready", timeout=2) as resp:
ready = resp.status == 200
except (urllib.error.URLError, TimeoutError, OSError):
ready = False
last = f"api_health={health} ready={ready}"
if len(health) == replicas and all(item == "healthy" for item in health) and ready:
return
time.sleep(2)
raise SystemExit("stack did not become ready: " + last)
def container_ids(env, service):
result = compose(["ps", "-q", service], env, capture=True)
if result.returncode != 0:
raise SystemExit(f"compose ps {service} failed")
return [line for line in result.stdout.split() if line]
def inspect_container(cid):
raw = subprocess.check_output(["docker", "inspect", cid], text=True)
return json.loads(raw)[0]
def env_map(info):
found = {}
for item in info["Config"].get("Env") or []:
key, _, value = item.partition("=")
found[key] = value
return found
def verify_limits(env, replicas, api_cpu, api_mem, db_cpu):
expect_api_nanos = cpu_nanos(api_cpu)
expect_api_mem = mem_bytes(api_mem)
expect_db_nanos = cpu_nanos(db_cpu)
expect_lb_nanos = cpu_nanos(LB_CPU_LIMIT)
api_ids = container_ids(env, "api")
if len(api_ids) != replicas:
raise SystemExit(f"expected {replicas} api containers, found {len(api_ids)}")
api_rows = []
for cid in api_ids:
info = inspect_container(cid)
nanos = info["HostConfig"]["NanoCpus"]
memory = info["HostConfig"]["Memory"]
present = env_map(info)
name = info["Name"].lstrip("/")
if nanos != expect_api_nanos or memory != expect_api_mem:
raise SystemExit(
f"{name}: NanoCpus={nanos} Memory={memory}, "
f"expected {expect_api_nanos} and {expect_api_mem}"
)
if present.get("HTTP_WORKERS") != HTTP_WORKERS or present.get("DB_POOL_SIZE") != DB_POOL_SIZE:
raise SystemExit(
f"{name}: HTTP_WORKERS={present.get('HTTP_WORKERS')} "
f"DB_POOL_SIZE={present.get('DB_POOL_SIZE')}"
)
api_rows.append(
{
"name": name,
"nano_cpus": nanos,
"memory": memory,
"http_workers": present["HTTP_WORKERS"],
"db_pool_size": present["DB_POOL_SIZE"],
}
)
total_mem = sum(row["memory"] for row in api_rows)
if total_mem != TOTAL_API_RAM:
raise SystemExit(f"api RAM sum {total_mem} != {TOTAL_API_RAM}")
db_ids = container_ids(env, "db")
if len(db_ids) != 1:
raise SystemExit(f"expected 1 db container, found {len(db_ids)}")
db_info = inspect_container(db_ids[0])
db_nanos = db_info["HostConfig"]["NanoCpus"]
if db_nanos != expect_db_nanos:
raise SystemExit(f"db NanoCpus={db_nanos}, expected {expect_db_nanos}")
lb_ids = container_ids(env, "lb")
if len(lb_ids) != 1:
raise SystemExit(f"expected 1 lb container, found {len(lb_ids)}")
lb_info = inspect_container(lb_ids[0])
lb_nanos = lb_info["HostConfig"]["NanoCpus"]
lb_env = env_map(lb_info)
if lb_nanos != expect_lb_nanos or lb_env.get("NGINX_WORKERS") != NGINX_WORKERS:
raise SystemExit(
f"lb NanoCpus={lb_nanos} NGINX_WORKERS={lb_env.get('NGINX_WORKERS')}"
)
rendered = compose(["exec", "-T", "lb", "nginx", "-T"], env, capture=True)
if rendered.returncode != 0:
raise SystemExit("nginx -T failed")
directive = ""
rendered_text = (rendered.stdout or "") + "\n" + (rendered.stderr or "")
for line in rendered_text.splitlines():
stripped = line.strip()
if stripped.startswith("worker_processes"):
directive = stripped
break
expected_directive = f"worker_processes {NGINX_WORKERS};"
if directive != expected_directive:
raise SystemExit(f"nginx config has {directive!r}, expected {expected_directive!r}")
logs = compose(["logs", "api"], env, capture=True)
if logs.returncode != 0:
raise SystemExit("api logs failed")
pool_line = f"pg pool ready ({DB_POOL_SIZE} connections)"
worker_line = f"http workers: {HTTP_WORKERS}"
if logs.stdout.count(pool_line) < replicas or logs.stdout.count(worker_line) < replicas:
raise SystemExit("api logs do not show the base pool and worker profile on every replica")
return {
"verified": True,
"api": api_rows,
"api_memory_sum": total_mem,
"db": {"name": db_info["Name"].lstrip("/"), "nano_cpus": db_nanos},
"lb": {
"name": lb_info["Name"].lstrip("/"),
"nano_cpus": lb_nanos,
"nginx_worker_processes": directive,
},
}
def sample_stats(stop, bucket):
while not stop.is_set():
proc = subprocess.run(
["docker", "stats", "--no-stream", "--format", "{{.Name}}\t{{.CPUPerc}}\t{{.MemUsage}}"],
text=True,
capture_output=True,
check=False,
)
stamp = time.time()
rows = []
for line in proc.stdout.splitlines():
parts = line.split("\t")
if len(parts) != 3:
continue
name, cpu, mem = parts
try:
cpu_num = float(cpu.strip().rstrip("%"))
except ValueError:
continue
rows.append({"name": name, "cpu_percent": cpu_num, "mem": mem.strip()})
if rows:
bucket.append({"t": stamp, "rows": rows})
stop.wait(3)
def summarize_cpu(samples):
acc = {}
for sample in samples:
for row in sample["rows"]:
name = row["name"].lstrip("/")
service = "other"
for prefix in ("api", "db", "cache", "lb"):
if f"-{prefix}-" in name or name.endswith(prefix):
service = prefix
break
acc.setdefault(service, []).append(row["cpu_percent"])
out = {}
for service, values in acc.items():
out[service] = {
"samples": len(values),
"mean_cpu_percent": round(sum(values) / len(values), 2),
"max_cpu_percent": round(max(values), 2),
}
return out
def run_point(series, replicas, api_cpu, api_mem, db_cpu):
env = env_for(replicas, api_cpu, api_mem, db_cpu)
label = f"track1-{series}-r{replicas}-apicpu{api_cpu}-dbcpu{db_cpu}"
raw = ROOT / "evidence" / f"{label}.load.json"
dest = ROOT / "evidence" / f"{label}.json"
print(
f"point series={series} replicas={replicas} api_cpu={api_cpu} "
f"api_mem={api_mem} db_cpu={db_cpu} http_workers={HTTP_WORKERS} "
f"db_pool={DB_POOL_SIZE} nginx_workers={NGINX_WORKERS}",
file=sys.stderr,
)
down = compose(["down"], env)
if down.returncode != 0:
raise SystemExit(down.returncode)
up = compose(
["up", "-d", "--build", "--scale", f"api={replicas}"],
env,
)
if up.returncode != 0:
raise SystemExit(up.returncode)
wait_stack(env, replicas)
limits = verify_limits(env, replicas, api_cpu, api_mem, db_cpu)
print(
f"inspect ok api_sum_memory={limits['api_memory_sum']} "
f"api_nano={limits['api'][0]['nano_cpus']} db_nano={limits['db']['nano_cpus']} "
f"nginx={limits['lb']['nginx_worker_processes']}",
file=sys.stderr,
)
warmup = subprocess.run(
[
sys.executable,
str(ROOT / "tests" / "load.py"),
"--duration",
"10",
"--concurrency",
"50",
"--output",
f"/tmp/{label}-warmup.json",
],
cwd=ROOT,
text=True,
capture_output=True,
)
if warmup.returncode != 0:
sys.stderr.write(warmup.stdout)
sys.stderr.write(warmup.stderr)
raise SystemExit("warmup failed")
samples = []
stop = threading.Event()
thread = threading.Thread(target=sample_stats, args=(stop, samples))
thread.start()
try:
measured = subprocess.run(
[
sys.executable,
str(ROOT / "tests" / "load.py"),
"--duration",
"30",
"--concurrency",
"50",
"--output",
str(raw),
],
cwd=ROOT,
text=True,
capture_output=True,
)
finally:
stop.set()
thread.join()
if measured.returncode != 0:
sys.stderr.write(measured.stdout)
sys.stderr.write(measured.stderr)
raise SystemExit("measured run failed")
load = json.loads(raw.read_text())
report = {
"series": series,
"api_replicas": replicas,
"api_cpu_limit": float(api_cpu),
"api_mem_limit": api_mem,
"api_mem_limit_bytes": mem_bytes(api_mem),
"api_mem_total_bytes": TOTAL_API_RAM,
"db_cpu_limit": float(db_cpu),
"http_workers": int(HTTP_WORKERS),
"pg_pool": int(DB_POOL_SIZE),
"nginx_worker_processes": NGINX_WORKERS,
"lb_cpu_limit": float(LB_CPU_LIMIT),
"warmup_seconds": 10,
"inspect": limits,
"cpu": summarize_cpu(samples),
"load": load,
}
dest.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n")
lat = load["successful_latency_ms"]
summary = {
"file": str(dest.relative_to(ROOT)),
"rps": load["successful_rps"],
"p50": lat["p50"],
"p95": lat["p95"],
"p99": lat["p99"],
"errors": load["failed_requests"],
"error_ratio": load["error_ratio"],
"api_mem_total_bytes": TOTAL_API_RAM,
"cpu": report["cpu"],
}
print(json.dumps(summary, ensure_ascii=False))
return summary
def main():
if len(sys.argv) != 1:
raise SystemExit("usage: python3 scripts/run_track1.py")
for series, replicas, api_cpu, api_mem, db_cpu in series_points():
run_point(series, replicas, api_cpu, api_mem, db_cpu)
if __name__ == "__main__":
main()