forked from mtusi2026/mtusi2026lab3
255 lines
8.8 KiB
Python
255 lines
8.8 KiB
Python
import argparse
|
|
import asyncio
|
|
import fcntl
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import platform
|
|
import random
|
|
import sys
|
|
import time
|
|
from datetime import datetime, timezone
|
|
|
|
from common import (
|
|
CONFIGS,
|
|
EVIDENCE,
|
|
HERE,
|
|
PRIVATE,
|
|
ROOT,
|
|
code_hash,
|
|
compose,
|
|
current_containers,
|
|
instrument_hash,
|
|
prepare_private,
|
|
restore_fixture,
|
|
shell,
|
|
write_json,
|
|
)
|
|
from load import measure
|
|
|
|
|
|
def protocol_hash(protocol):
|
|
return hashlib.sha256(
|
|
(json.dumps(protocol, ensure_ascii=False, indent=2) + "\n").encode()
|
|
).hexdigest()
|
|
|
|
|
|
def main_compose(args):
|
|
return shell(
|
|
[
|
|
"docker",
|
|
"compose",
|
|
"--project-name",
|
|
"highload-lab3",
|
|
"--env-file",
|
|
".env",
|
|
"-f",
|
|
"compose.yml",
|
|
*args,
|
|
]
|
|
)
|
|
|
|
|
|
def setup():
|
|
prepare_private()
|
|
main_ids = main_compose(["ps", "-q"]).split()
|
|
# Preserve the original state so completion/error restores only our main app.
|
|
if not PRIVATE.joinpath("original-state.json").exists():
|
|
write_json(PRIVATE / "original-state.json", {"was_running": bool(main_ids)})
|
|
main_compose(["stop"])
|
|
compose(["up", "-d", "--wait"])
|
|
write_json(
|
|
EVIDENCE / "environment.json",
|
|
{
|
|
"time": datetime.now(timezone.utc).isoformat(),
|
|
"platform": platform.platform(),
|
|
"python": sys.version,
|
|
"cpu_count_wsl": os.cpu_count(),
|
|
"docker": shell(["docker", "version", "--format", "{{json .Server}}"]),
|
|
"docker_resources": shell(
|
|
["docker", "info", "--format", "{{.NCPU}} {{.MemTotal}}"]
|
|
),
|
|
"compose": shell(["docker", "compose", "version"]),
|
|
"git_commit": shell(["git", "rev-parse", "HEAD"]),
|
|
"code_sha256": code_hash(),
|
|
"fixture_sha256": hashlib.sha256(
|
|
HERE.joinpath("fixture.sql").read_bytes()
|
|
).hexdigest(),
|
|
"dependencies": shell([sys.executable, "-m", "pip", "freeze"]),
|
|
"main_app_stopped_with_volumes_preserved": True,
|
|
},
|
|
)
|
|
|
|
|
|
def restore_main():
|
|
compose(["stop"])
|
|
path = PRIVATE / "original-state.json"
|
|
if path.exists() and json.loads(path.read_text())["was_running"]:
|
|
main_compose(["up", "-d", "--wait"])
|
|
print("Restored original application state; benchmark volumes retained", flush=True)
|
|
|
|
|
|
def run_one(series, config_name, rate, repeat, protocol, *, pilot=False):
|
|
config = dict(
|
|
CONFIGS[protocol.get("config_bases", {}).get(config_name, config_name)]
|
|
)
|
|
if "config_overrides" in protocol:
|
|
config.update(protocol["config_overrides"].get(config_name, {}))
|
|
# Reset Go heap, goroutines, prepared-statement caches and connection pools
|
|
# before EVERY point, including adjacent rates of the same configuration.
|
|
compose(["up", "-d", "--no-deps", "--force-recreate", "--wait", "api"], config)
|
|
actors = restore_fixture()
|
|
run_id = f"{config_name}-r{repeat}-rate{rate}"
|
|
directory = EVIDENCE / series / run_id
|
|
if directory.exists():
|
|
if directory.joinpath("summary.json").exists():
|
|
saved = json.loads(directory.joinpath("summary.json").read_text())
|
|
manifest = json.loads(directory.joinpath("manifest.json").read_text())
|
|
if manifest["protocol_sha256"] != protocol_hash(protocol):
|
|
raise RuntimeError("Cannot reuse a run from a different protocol")
|
|
print("Already completed " + run_id, flush=True)
|
|
return saved
|
|
raise RuntimeError(
|
|
"Incomplete run exists; retain it and choose a new series name"
|
|
)
|
|
directory.mkdir(parents=True)
|
|
actual = current_containers()
|
|
apis = [row for row in actual if row["service"] == "api"]
|
|
if len(apis) != config["replicas"] or any(
|
|
row["cpu"] != config["cpu"] * 1e9
|
|
or row["memory"] != int(config["memory"].removesuffix("m")) * 1048576
|
|
or row["ports"]
|
|
for row in apis
|
|
):
|
|
raise RuntimeError(
|
|
"Effective API resources/ports differ from the planned configuration"
|
|
)
|
|
write_json(
|
|
directory / "manifest.json",
|
|
{
|
|
"run_id": run_id,
|
|
"series": series,
|
|
"pilot": pilot,
|
|
"config_name": config_name,
|
|
"config": config,
|
|
"rate": rate,
|
|
"repeat": repeat,
|
|
"workload_seed": protocol["workload_seed"],
|
|
"protocol_sha256": protocol_hash(protocol),
|
|
"code_sha256": code_hash(),
|
|
"containers": actual,
|
|
"instrument_sha256": instrument_hash(),
|
|
"started_utc": datetime.now(timezone.utc).isoformat(),
|
|
"fixture_users": 200,
|
|
"fixture_chats": 100,
|
|
"fixture_messages": 10000,
|
|
},
|
|
)
|
|
summary = asyncio.run(
|
|
measure(
|
|
actors,
|
|
directory,
|
|
config,
|
|
rate=rate,
|
|
duration=protocol["duration_s"],
|
|
warmup=protocol["warmup_s"],
|
|
seed=protocol["workload_seed"],
|
|
prefix=series + ":" + run_id,
|
|
limit=protocol["inflight_limit"],
|
|
)
|
|
)
|
|
print(
|
|
json.dumps(
|
|
{
|
|
"run": run_id,
|
|
"success_ratio": round(summary["success_ratio"], 4),
|
|
"p95_ms": summary["p95_ms"],
|
|
"goodput_rps": round(summary["goodput_rps"], 2),
|
|
"generator_cpu": round(summary["generator_cpu_cores"], 3),
|
|
"dispatch_p95_ms": summary["dispatch_p95_ms"],
|
|
"valid": summary["valid"],
|
|
"slo_pass": summary["slo_pass"],
|
|
}
|
|
),
|
|
flush=True,
|
|
)
|
|
return summary
|
|
|
|
|
|
def run_series(series, protocol, pilot=False):
|
|
if not pilot:
|
|
expected = protocol["code_sha256"]
|
|
if code_hash() != expected:
|
|
raise RuntimeError("Baseline code differs from the frozen protocol")
|
|
if instrument_hash() != protocol["instrument_sha256"]:
|
|
raise RuntimeError("Instruments differ from the frozen protocol")
|
|
order = []
|
|
for repeat, seed in enumerate(protocol["order_seeds"], 1):
|
|
names = list(protocol["configs"])
|
|
random.Random(seed).shuffle(names)
|
|
for name in names:
|
|
for rate in protocol["rates"]:
|
|
order.append({"config": name, "rate": rate, "repeat": repeat})
|
|
directory = EVIDENCE / series
|
|
directory.mkdir(parents=True, exist_ok=True)
|
|
frozen = directory / "protocol.json"
|
|
if frozen.exists() and json.loads(frozen.read_text()) != protocol:
|
|
raise RuntimeError("Series protocol already exists and differs")
|
|
write_json(frozen, protocol)
|
|
directory.joinpath("protocol.sha256").write_text(
|
|
hashlib.sha256(frozen.read_bytes()).hexdigest() + " protocol.json\n"
|
|
)
|
|
# Capture the complete non-secret implementation, including uncommitted files.
|
|
import tarfile
|
|
|
|
archive = directory / "source.tar.gz"
|
|
if not archive.exists():
|
|
with tarfile.open(archive, "w:gz") as tar:
|
|
for base in (ROOT / "src/backend", HERE):
|
|
for path in sorted(base.rglob("*")):
|
|
if path.is_file() and "__pycache__" not in path.parts:
|
|
tar.add(path, arcname=str(path.relative_to(ROOT)))
|
|
for path in (
|
|
ROOT / "src/go.mod",
|
|
ROOT / "src/go.sum",
|
|
ROOT / "src/Dockerfile",
|
|
):
|
|
tar.add(path, arcname=str(path.relative_to(ROOT)))
|
|
write_json(directory / "order.json", order)
|
|
print(f"Starting {series}: {len(order)} runs, pilot={pilot}", flush=True)
|
|
for index, item in enumerate(order, 1):
|
|
print(f"Progress {index}/{len(order)}", flush=True)
|
|
run_one(
|
|
series, item["config"], item["rate"], item["repeat"], protocol, pilot=pilot
|
|
)
|
|
time.sleep(protocol["cooldown_s"])
|
|
print("SERIES COMPLETE: " + series, flush=True)
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(__doc__)
|
|
parser.add_argument("action", choices=("prepare", "pilot", "series", "restore"))
|
|
parser.add_argument("--protocol", default=None)
|
|
parser.add_argument("--series", default=None)
|
|
args = parser.parse_args()
|
|
prepare_private()
|
|
with PRIVATE.joinpath("experiment.lock").open("w") as lock:
|
|
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
if args.action == "prepare":
|
|
setup()
|
|
elif args.action == "restore":
|
|
restore_main()
|
|
else:
|
|
if not args.protocol or not args.series:
|
|
parser.error("pilot/series require --protocol and --series")
|
|
protocol = json.loads(ROOT.joinpath(args.protocol).read_text())
|
|
try:
|
|
run_series(args.series, protocol, pilot=args.action == "pilot")
|
|
except BaseException:
|
|
restore_main()
|
|
raise
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|