226 lines
7.7 KiB
Python
226 lines
7.7 KiB
Python
"""Reproducible scaling matrix with actual quotas and service resource samples."""
|
|
|
|
import argparse
|
|
import asyncio
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import platform
|
|
import subprocess
|
|
import sys
|
|
import threading
|
|
import time
|
|
from datetime import UTC, datetime
|
|
from pathlib import Path
|
|
|
|
from load import run_load
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT / "tests"))
|
|
from client import Client # noqa: E402
|
|
|
|
|
|
def command(args, *, env=None):
|
|
return subprocess.check_output(args, cwd=ROOT, env=env, text=True)
|
|
|
|
|
|
def inventory():
|
|
ids = command(["docker", "compose", "ps", "-q"]).split()
|
|
result = []
|
|
for cid in ids:
|
|
raw = json.loads(command(["docker", "inspect", cid]))[0]
|
|
result.append(
|
|
{
|
|
"id": cid,
|
|
"service": raw["Config"]["Labels"]["com.docker.compose.service"],
|
|
"image_id": raw["Image"],
|
|
"image": raw["Config"]["Image"],
|
|
"nano_cpus": raw["HostConfig"]["NanoCpus"],
|
|
"memory_bytes": raw["HostConfig"]["Memory"],
|
|
"started_at": raw["State"]["StartedAt"],
|
|
}
|
|
)
|
|
return result
|
|
|
|
|
|
def sample_services(stop, samples, containers):
|
|
while not stop.is_set():
|
|
raw = command(
|
|
[
|
|
"docker",
|
|
"stats",
|
|
"--no-stream",
|
|
"--format",
|
|
"{{json .}}",
|
|
*[c["id"] for c in containers],
|
|
]
|
|
)
|
|
rows = [json.loads(line) for line in raw.splitlines() if line.strip()]
|
|
samples.append({"monotonic": time.monotonic(), "containers": rows})
|
|
stop.wait(0.5)
|
|
|
|
|
|
def configure(profile):
|
|
env = {
|
|
**os.environ,
|
|
"API_REPLICAS": str(profile["replicas"]),
|
|
"API_CPUS": str(profile["api_cpu"]),
|
|
"API_MEMORY": profile["api_memory"],
|
|
"PG_CPUS": str(profile["pg_cpu"]),
|
|
"PG_MEMORY": "768m",
|
|
}
|
|
subprocess.run(
|
|
["docker", "compose", "up", "-d", "--scale", "api=" + str(profile["replicas"]), "--wait"],
|
|
cwd=ROOT,
|
|
env=env,
|
|
check=True,
|
|
stdout=subprocess.DEVNULL,
|
|
)
|
|
subprocess.run(
|
|
[sys.executable, "scripts/wait_replicas.py", str(profile["replicas"])], cwd=ROOT, check=True
|
|
)
|
|
actual = inventory()
|
|
apis = [c for c in actual if c["service"] == "api"]
|
|
assert len(apis) == profile["replicas"]
|
|
assert all(c["nano_cpus"] == int(profile["api_cpu"] * 1e9) for c in apis)
|
|
assert all(c["memory_bytes"] == int(profile["api_memory"][:-1]) * 1024**2 for c in apis)
|
|
db = next(c for c in actual if c["service"] == "postgres")
|
|
assert db["nano_cpus"] == int(profile["pg_cpu"] * 1e9)
|
|
return actual
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("--duration", type=float, default=6)
|
|
parser.add_argument("--rates", default="100,300,600")
|
|
parser.add_argument("--repeats", type=int, default=3)
|
|
parser.add_argument("--output", default="evidence/scaling.json")
|
|
args = parser.parse_args()
|
|
profiles = []
|
|
for replicas in [1, 2, 4]:
|
|
profiles.append(
|
|
{
|
|
"name": f"fixed-{replicas}",
|
|
"series": "fixed",
|
|
"replicas": replicas,
|
|
"api_cpu": 1 / replicas,
|
|
"api_memory": f"{1024 // replicas}m",
|
|
"pg_cpu": 0.25,
|
|
}
|
|
)
|
|
for replicas in [1, 2, 4]:
|
|
profiles.append(
|
|
{
|
|
"name": f"growth-{replicas}",
|
|
"series": "growth",
|
|
"replicas": replicas,
|
|
"api_cpu": 0.5,
|
|
"api_memory": "256m",
|
|
"pg_cpu": 0.25,
|
|
}
|
|
)
|
|
profiles.append(
|
|
{
|
|
"name": "db-improved",
|
|
"series": "improvement",
|
|
"replicas": 4,
|
|
"api_cpu": 0.5,
|
|
"api_memory": "256m",
|
|
"pg_cpu": 0.5,
|
|
}
|
|
)
|
|
source_manifest = {
|
|
str(p.relative_to(ROOT)): hashlib.sha256(p.read_bytes()).hexdigest()
|
|
for folder in ["src", "migrations"]
|
|
for p in sorted((ROOT / folder).rglob("*"))
|
|
if p.is_file() and "__pycache__" not in p.parts
|
|
}
|
|
report = {
|
|
"source_manifest": source_manifest,
|
|
"source_revision": command(["git", "rev-parse", "HEAD"]).strip(),
|
|
"started_at": datetime.now(UTC).isoformat(),
|
|
"host": platform.platform(),
|
|
"docker": json.loads(command(["docker", "info", "--format", "{{json .}}"])),
|
|
"profiles": profiles,
|
|
"runs": [],
|
|
"notes": [
|
|
"PostgreSQL intentionally constrained to 0.25CPU to expose its resource boundary.",
|
|
"Open-loop scheduler drops if concurrency cap128 or lag>250ms; no hidden backlog.",
|
|
"Only successes enter latency percentiles; wall time includes draining requests.",
|
|
],
|
|
}
|
|
# Keep only non-sensitive engine inventory fields.
|
|
report["docker"] = {
|
|
k: report["docker"].get(k)
|
|
for k in ["ServerVersion", "NCPU", "MemTotal", "Architecture", "OperatingSystem"]
|
|
}
|
|
dest = ROOT / args.output
|
|
dest.parent.mkdir(parents=True, exist_ok=True)
|
|
try:
|
|
c = Client("http://127.0.0.1:8083")
|
|
actor = c.user("study")
|
|
for profile in profiles:
|
|
actual = configure(profile)
|
|
for repeat in range(args.repeats):
|
|
rates = [int(x) for x in args.rates.split(",")]
|
|
if repeat % 2:
|
|
rates.reverse()
|
|
for rate in rates:
|
|
chat = c.call(
|
|
"POST",
|
|
"/api/v1/chats",
|
|
201,
|
|
actor=actor,
|
|
body={"title": "synthetic-study", "member_ids": [actor["id"]]},
|
|
schema="Chat",
|
|
)["json"]
|
|
samples, stop = [], threading.Event()
|
|
thread = threading.Thread(
|
|
target=sample_services, args=(stop, samples, actual), daemon=True
|
|
)
|
|
thread.start()
|
|
result = asyncio.run(
|
|
run_load(c.base, actor, chat["id"], rate=rate, duration=args.duration)
|
|
)
|
|
stop.set()
|
|
thread.join()
|
|
result.update(
|
|
{
|
|
"configuration": profile["name"],
|
|
"repeat": repeat + 1,
|
|
"actual_containers": actual,
|
|
"resource_samples": samples,
|
|
}
|
|
)
|
|
report["runs"].append(result)
|
|
dest.write_text(json.dumps(report, indent=2) + "\n")
|
|
print(
|
|
json.dumps(
|
|
{
|
|
k: result[k]
|
|
for k in [
|
|
"configuration",
|
|
"repeat",
|
|
"offered_rate",
|
|
"successful_rps",
|
|
"errors",
|
|
"dropped",
|
|
]
|
|
}
|
|
),
|
|
flush=True,
|
|
)
|
|
finally:
|
|
subprocess.run(
|
|
["docker", "compose", "up", "-d", "--scale", "api=2", "--wait"],
|
|
cwd=ROOT,
|
|
check=True,
|
|
stdout=subprocess.DEVNULL,
|
|
)
|
|
report["ended_at"] = datetime.now(UTC).isoformat()
|
|
dest.write_text(json.dumps(report, indent=2) + "\n")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|