Files
mtusi2026lab3/research/run.py
T

226 lines
7.7 KiB
Python

"""Reproducible scaling matrix with actual quotas and service resource samples."""
import argparse
import asyncio
import hashlib
import json
import os
import platform
import subprocess
import sys
import threading
import time
from datetime import UTC, datetime
from pathlib import Path
from load import run_load
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
from client import Client # noqa: E402
def command(args, *, env=None):
return subprocess.check_output(args, cwd=ROOT, env=env, text=True)
def inventory():
ids = command(["docker", "compose", "ps", "-q"]).split()
result = []
for cid in ids:
raw = json.loads(command(["docker", "inspect", cid]))[0]
result.append(
{
"id": cid,
"service": raw["Config"]["Labels"]["com.docker.compose.service"],
"image_id": raw["Image"],
"image": raw["Config"]["Image"],
"nano_cpus": raw["HostConfig"]["NanoCpus"],
"memory_bytes": raw["HostConfig"]["Memory"],
"started_at": raw["State"]["StartedAt"],
}
)
return result
def sample_services(stop, samples, containers):
while not stop.is_set():
raw = command(
[
"docker",
"stats",
"--no-stream",
"--format",
"{{json .}}",
*[c["id"] for c in containers],
]
)
rows = [json.loads(line) for line in raw.splitlines() if line.strip()]
samples.append({"monotonic": time.monotonic(), "containers": rows})
stop.wait(0.5)
def configure(profile):
env = {
**os.environ,
"API_REPLICAS": str(profile["replicas"]),
"API_CPUS": str(profile["api_cpu"]),
"API_MEMORY": profile["api_memory"],
"PG_CPUS": str(profile["pg_cpu"]),
"PG_MEMORY": "768m",
}
subprocess.run(
["docker", "compose", "up", "-d", "--scale", "api=" + str(profile["replicas"]), "--wait"],
cwd=ROOT,
env=env,
check=True,
stdout=subprocess.DEVNULL,
)
subprocess.run(
[sys.executable, "scripts/wait_replicas.py", str(profile["replicas"])], cwd=ROOT, check=True
)
actual = inventory()
apis = [c for c in actual if c["service"] == "api"]
assert len(apis) == profile["replicas"]
assert all(c["nano_cpus"] == int(profile["api_cpu"] * 1e9) for c in apis)
assert all(c["memory_bytes"] == int(profile["api_memory"][:-1]) * 1024**2 for c in apis)
db = next(c for c in actual if c["service"] == "postgres")
assert db["nano_cpus"] == int(profile["pg_cpu"] * 1e9)
return actual
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--duration", type=float, default=6)
parser.add_argument("--rates", default="100,300,600")
parser.add_argument("--repeats", type=int, default=3)
parser.add_argument("--output", default="evidence/scaling.json")
args = parser.parse_args()
profiles = []
for replicas in [1, 2, 4]:
profiles.append(
{
"name": f"fixed-{replicas}",
"series": "fixed",
"replicas": replicas,
"api_cpu": 1 / replicas,
"api_memory": f"{1024 // replicas}m",
"pg_cpu": 0.25,
}
)
for replicas in [1, 2, 4]:
profiles.append(
{
"name": f"growth-{replicas}",
"series": "growth",
"replicas": replicas,
"api_cpu": 0.5,
"api_memory": "256m",
"pg_cpu": 0.25,
}
)
profiles.append(
{
"name": "db-improved",
"series": "improvement",
"replicas": 4,
"api_cpu": 0.5,
"api_memory": "256m",
"pg_cpu": 0.5,
}
)
source_manifest = {
str(p.relative_to(ROOT)): hashlib.sha256(p.read_bytes()).hexdigest()
for folder in ["src", "migrations"]
for p in sorted((ROOT / folder).rglob("*"))
if p.is_file() and "__pycache__" not in p.parts
}
report = {
"source_manifest": source_manifest,
"source_revision": command(["git", "rev-parse", "HEAD"]).strip(),
"started_at": datetime.now(UTC).isoformat(),
"host": platform.platform(),
"docker": json.loads(command(["docker", "info", "--format", "{{json .}}"])),
"profiles": profiles,
"runs": [],
"notes": [
"PostgreSQL intentionally constrained to 0.25CPU to expose its resource boundary.",
"Open-loop scheduler drops if concurrency cap128 or lag>250ms; no hidden backlog.",
"Only successes enter latency percentiles; wall time includes draining requests.",
],
}
# Keep only non-sensitive engine inventory fields.
report["docker"] = {
k: report["docker"].get(k)
for k in ["ServerVersion", "NCPU", "MemTotal", "Architecture", "OperatingSystem"]
}
dest = ROOT / args.output
dest.parent.mkdir(parents=True, exist_ok=True)
try:
c = Client("http://127.0.0.1:8083")
actor = c.user("study")
for profile in profiles:
actual = configure(profile)
for repeat in range(args.repeats):
rates = [int(x) for x in args.rates.split(",")]
if repeat % 2:
rates.reverse()
for rate in rates:
chat = c.call(
"POST",
"/api/v1/chats",
201,
actor=actor,
body={"title": "synthetic-study", "member_ids": [actor["id"]]},
schema="Chat",
)["json"]
samples, stop = [], threading.Event()
thread = threading.Thread(
target=sample_services, args=(stop, samples, actual), daemon=True
)
thread.start()
result = asyncio.run(
run_load(c.base, actor, chat["id"], rate=rate, duration=args.duration)
)
stop.set()
thread.join()
result.update(
{
"configuration": profile["name"],
"repeat": repeat + 1,
"actual_containers": actual,
"resource_samples": samples,
}
)
report["runs"].append(result)
dest.write_text(json.dumps(report, indent=2) + "\n")
print(
json.dumps(
{
k: result[k]
for k in [
"configuration",
"repeat",
"offered_rate",
"successful_rps",
"errors",
"dropped",
]
}
),
flush=True,
)
finally:
subprocess.run(
["docker", "compose", "up", "-d", "--scale", "api=2", "--wait"],
cwd=ROOT,
check=True,
stdout=subprocess.DEVNULL,
)
report["ended_at"] = datetime.now(UTC).isoformat()
dest.write_text(json.dumps(report, indent=2) + "\n")
if __name__ == "__main__":
main()