Files
mtusi2026lab3/experiments/lab3-scale/run.py
T
2026-10-10 21:32:33 +03:00

255 lines
8.8 KiB
Python

import argparse
import asyncio
import fcntl
import hashlib
import json
import os
import platform
import random
import sys
import time
from datetime import datetime, timezone
from common import (
CONFIGS,
EVIDENCE,
HERE,
PRIVATE,
ROOT,
code_hash,
compose,
current_containers,
instrument_hash,
prepare_private,
restore_fixture,
shell,
write_json,
)
from load import measure
def protocol_hash(protocol):
return hashlib.sha256(
(json.dumps(protocol, ensure_ascii=False, indent=2) + "\n").encode()
).hexdigest()
def main_compose(args):
return shell(
[
"docker",
"compose",
"--project-name",
"highload-lab3",
"--env-file",
".env",
"-f",
"compose.yml",
*args,
]
)
def setup():
prepare_private()
main_ids = main_compose(["ps", "-q"]).split()
# Preserve the original state so completion/error restores only our main app.
if not PRIVATE.joinpath("original-state.json").exists():
write_json(PRIVATE / "original-state.json", {"was_running": bool(main_ids)})
main_compose(["stop"])
compose(["up", "-d", "--wait"])
write_json(
EVIDENCE / "environment.json",
{
"time": datetime.now(timezone.utc).isoformat(),
"platform": platform.platform(),
"python": sys.version,
"cpu_count_wsl": os.cpu_count(),
"docker": shell(["docker", "version", "--format", "{{json .Server}}"]),
"docker_resources": shell(
["docker", "info", "--format", "{{.NCPU}} {{.MemTotal}}"]
),
"compose": shell(["docker", "compose", "version"]),
"git_commit": shell(["git", "rev-parse", "HEAD"]),
"code_sha256": code_hash(),
"fixture_sha256": hashlib.sha256(
HERE.joinpath("fixture.sql").read_bytes()
).hexdigest(),
"dependencies": shell([sys.executable, "-m", "pip", "freeze"]),
"main_app_stopped_with_volumes_preserved": True,
},
)
def restore_main():
compose(["stop"])
path = PRIVATE / "original-state.json"
if path.exists() and json.loads(path.read_text())["was_running"]:
main_compose(["up", "-d", "--wait"])
print("Restored original application state; benchmark volumes retained", flush=True)
def run_one(series, config_name, rate, repeat, protocol, *, pilot=False):
config = dict(
CONFIGS[protocol.get("config_bases", {}).get(config_name, config_name)]
)
if "config_overrides" in protocol:
config.update(protocol["config_overrides"].get(config_name, {}))
# Reset Go heap, goroutines, prepared-statement caches and connection pools
# before EVERY point, including adjacent rates of the same configuration.
compose(["up", "-d", "--no-deps", "--force-recreate", "--wait", "api"], config)
actors = restore_fixture()
run_id = f"{config_name}-r{repeat}-rate{rate}"
directory = EVIDENCE / series / run_id
if directory.exists():
if directory.joinpath("summary.json").exists():
saved = json.loads(directory.joinpath("summary.json").read_text())
manifest = json.loads(directory.joinpath("manifest.json").read_text())
if manifest["protocol_sha256"] != protocol_hash(protocol):
raise RuntimeError("Cannot reuse a run from a different protocol")
print("Already completed " + run_id, flush=True)
return saved
raise RuntimeError(
"Incomplete run exists; retain it and choose a new series name"
)
directory.mkdir(parents=True)
actual = current_containers()
apis = [row for row in actual if row["service"] == "api"]
if len(apis) != config["replicas"] or any(
row["cpu"] != config["cpu"] * 1e9
or row["memory"] != int(config["memory"].removesuffix("m")) * 1048576
or row["ports"]
for row in apis
):
raise RuntimeError(
"Effective API resources/ports differ from the planned configuration"
)
write_json(
directory / "manifest.json",
{
"run_id": run_id,
"series": series,
"pilot": pilot,
"config_name": config_name,
"config": config,
"rate": rate,
"repeat": repeat,
"workload_seed": protocol["workload_seed"],
"protocol_sha256": protocol_hash(protocol),
"code_sha256": code_hash(),
"containers": actual,
"instrument_sha256": instrument_hash(),
"started_utc": datetime.now(timezone.utc).isoformat(),
"fixture_users": 200,
"fixture_chats": 100,
"fixture_messages": 10000,
},
)
summary = asyncio.run(
measure(
actors,
directory,
config,
rate=rate,
duration=protocol["duration_s"],
warmup=protocol["warmup_s"],
seed=protocol["workload_seed"],
prefix=series + ":" + run_id,
limit=protocol["inflight_limit"],
)
)
print(
json.dumps(
{
"run": run_id,
"success_ratio": round(summary["success_ratio"], 4),
"p95_ms": summary["p95_ms"],
"goodput_rps": round(summary["goodput_rps"], 2),
"generator_cpu": round(summary["generator_cpu_cores"], 3),
"dispatch_p95_ms": summary["dispatch_p95_ms"],
"valid": summary["valid"],
"slo_pass": summary["slo_pass"],
}
),
flush=True,
)
return summary
def run_series(series, protocol, pilot=False):
if not pilot:
expected = protocol["code_sha256"]
if code_hash() != expected:
raise RuntimeError("Baseline code differs from the frozen protocol")
if instrument_hash() != protocol["instrument_sha256"]:
raise RuntimeError("Instruments differ from the frozen protocol")
order = []
for repeat, seed in enumerate(protocol["order_seeds"], 1):
names = list(protocol["configs"])
random.Random(seed).shuffle(names)
for name in names:
for rate in protocol["rates"]:
order.append({"config": name, "rate": rate, "repeat": repeat})
directory = EVIDENCE / series
directory.mkdir(parents=True, exist_ok=True)
frozen = directory / "protocol.json"
if frozen.exists() and json.loads(frozen.read_text()) != protocol:
raise RuntimeError("Series protocol already exists and differs")
write_json(frozen, protocol)
directory.joinpath("protocol.sha256").write_text(
hashlib.sha256(frozen.read_bytes()).hexdigest() + " protocol.json\n"
)
# Capture the complete non-secret implementation, including uncommitted files.
import tarfile
archive = directory / "source.tar.gz"
if not archive.exists():
with tarfile.open(archive, "w:gz") as tar:
for base in (ROOT / "src/backend", HERE):
for path in sorted(base.rglob("*")):
if path.is_file() and "__pycache__" not in path.parts:
tar.add(path, arcname=str(path.relative_to(ROOT)))
for path in (
ROOT / "src/go.mod",
ROOT / "src/go.sum",
ROOT / "src/Dockerfile",
):
tar.add(path, arcname=str(path.relative_to(ROOT)))
write_json(directory / "order.json", order)
print(f"Starting {series}: {len(order)} runs, pilot={pilot}", flush=True)
for index, item in enumerate(order, 1):
print(f"Progress {index}/{len(order)}", flush=True)
run_one(
series, item["config"], item["rate"], item["repeat"], protocol, pilot=pilot
)
time.sleep(protocol["cooldown_s"])
print("SERIES COMPLETE: " + series, flush=True)
def main():
parser = argparse.ArgumentParser(__doc__)
parser.add_argument("action", choices=("prepare", "pilot", "series", "restore"))
parser.add_argument("--protocol", default=None)
parser.add_argument("--series", default=None)
args = parser.parse_args()
prepare_private()
with PRIVATE.joinpath("experiment.lock").open("w") as lock:
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
if args.action == "prepare":
setup()
elif args.action == "restore":
restore_main()
else:
if not args.protocol or not args.series:
parser.error("pilot/series require --protocol and --series")
protocol = json.loads(ROOT.joinpath(args.protocol).read_text())
try:
run_series(args.series, protocol, pilot=args.action == "pilot")
except BaseException:
restore_main()
raise
if __name__ == "__main__":
main()