Files
mtusi2026lab3/scripts/benchmark.py
T

323 lines
15 KiB
Python

#!/usr/bin/env python3
"""Reproducible, rate-controlled GET benchmark for the lab3 scaling track."""
import argparse
from collections import defaultdict
import http.client
import json
import math
import os
from pathlib import Path
from queue import Full, Queue
import statistics
import subprocess
import sys
import tempfile
import threading
import time
from urllib.parse import urlsplit
from urllib.request import urlopen
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
from client import Client # noqa: E402 - the repository's public bootstrap client
def topology(mode, replicas):
if mode not in ("fixed", "growth") or replicas not in (1, 2, 4):
raise ValueError("mode must be fixed/growth and replicas must be 1/2/4")
return replicas, (1 / replicas if mode == "fixed" else 0.5), (512 // replicas if mode == "fixed" else 256)
def validate_matrix(rows):
if not rows:
raise ValueError("no benchmark rows")
rates = {row["rate"] for row in rows}
if len(rates) < 3:
raise ValueError("need at least three offered rates")
profiles = {(row["duration"], row["warmup"], row["dataset_messages"]) for row in rows}
if len(profiles) != 1:
raise ValueError("workload profile changed across measurements")
for mode in ("fixed", "growth"):
for replicas in (1, 2, 4):
_, cpu, memory = topology(mode, replicas)
for rate in rates:
repeats = [row["repeat"] for row in rows if (row["mode"], row["replicas"], row["rate"]) == (mode, replicas, rate)]
if sorted(repeats) != [1, 2, 3]:
raise ValueError(f"need three repeats for {mode}/{replicas}/{rate}")
for row in rows:
if (row["mode"], row["replicas"]) == (mode, replicas) and (row["api_cpu_each"], row["api_memory_mib_each"]) != (cpu, memory):
raise ValueError("incorrect API resource quota")
def percentile(values, fraction):
if not values:
return None
ordered = sorted(values)
return round(ordered[math.ceil(fraction * len(ordered)) - 1], 3)
def command(*args):
return subprocess.run(args, cwd=ROOT, text=True, capture_output=True, check=True).stdout.strip()
def compose(*args, override=None):
prefix = ["docker", "compose", "--env-file", ".env", "-f", "compose.yaml"]
if override:
prefix += ["-f", str(override)]
return command(*prefix, *args)
def pool_snapshot(base_url, replicas):
found = {}
for _ in range(8 * replicas):
with urlopen(base_url + "/health/ready", timeout=4) as response:
headers = response.headers
found[headers["X-Instance-Id"]] = {
"requests": int(headers["X-Pool-Requests"]),
"queued": int(headers["X-Pool-Queued"]),
"wait_ms": int(headers["X-Pool-Wait-Ms"]),
"size": int(headers["X-Pool-Size"]),
"available": int(headers["X-Pool-Available"]),
}
if len(found) == replicas:
break
return found
def db_snapshot(override):
sql = ("SELECT pg_database_size(current_database()),xact_commit,xact_rollback,"
"blks_read,blks_hit,temp_files,deadlocks,(SELECT count(*) FROM messages) "
"FROM pg_stat_database WHERE datname=current_database()")
values = command("docker", "compose", "--env-file", ".env", "-f", "compose.yaml", "-f", str(override),
"exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", "-At", "-F", "|", "-c", sql)
return dict(zip(("database_bytes", "commits", "rollbacks", "blocks_read", "blocks_hit", "temp_files", "deadlocks", "messages"),
map(int, values.split("|"))))
def docker_ids(override):
ids = {}
for service in ("api", "postgres", "lb"):
ids[service] = compose("ps", "-q", service, override=override).splitlines()
return ids
def stats_sample(ids):
all_ids = [id for service_ids in ids.values() for id in service_ids]
if not all_ids:
return {}
output = command("docker", "stats", "--no-stream", "--format", "{{json .}}", *all_ids)
by_id = {}
for line in output.splitlines():
item = json.loads(line)
by_id[item["ID"]] = item
result = {}
for service, service_ids in ids.items():
cpu = []
memory = []
for id in service_ids:
item = by_id.get(id[:12]) or by_id.get(id)
if not item:
raise ValueError("Docker stats omitted a benchmark container")
cpu.append(float(item["CPUPerc"].rstrip("%")))
amount = item["MemUsage"].split("/")[0].strip()
value, unit = float("".join(c for c in amount if c.isdigit() or c == ".")), "".join(c for c in amount if c.isalpha())
memory.append(round(value * {"B": 1 / 2**20, "KiB": 1 / 1024, "MiB": 1, "GiB": 1024,
"kB": 1000 / 2**20, "MB": 10**6 / 2**20, "GB": 10**9 / 2**20}[unit], 3))
result[service] = {"cpu_percent": round(sum(cpu), 3), "memory_mib": round(sum(memory), 3)}
return result
def open_loop(base_url, path, token, rate, duration, workers=64):
"""Schedule arrivals by clock; latency includes queue wait and HTTP time."""
address = urlsplit(base_url)
if address.scheme != "http":
raise ValueError("this local lab3 benchmark expects HTTP")
jobs = Queue(maxsize=256)
latencies = []
errors = defaultdict(int)
lock = threading.Lock()
starts = time.monotonic()
cpu_start = time.process_time()
count = math.ceil(rate * duration)
def worker():
connection = None
while True:
due = jobs.get()
if due is None:
jobs.task_done()
break
try:
if connection is None:
connection = http.client.HTTPConnection(address.hostname, address.port or 80, timeout=4)
connection.request("GET", path, headers={"Authorization": "Bearer " + token, "Accept": "application/json"})
response = connection.getresponse()
payload = response.read()
if response.status != 200:
with lock:
errors[f"http_{response.status}"] += 1
elif len(json.loads(payload)["items"]) != 20:
with lock:
errors["wrong_page"] += 1
else:
with lock:
latencies.append((time.monotonic() - due) * 1000)
except (OSError, TimeoutError, ValueError, KeyError, http.client.HTTPException):
with lock:
errors["transport_or_response"] += 1
if connection:
connection.close()
connection = None
finally:
jobs.task_done()
if connection:
connection.close()
threads = [threading.Thread(target=worker, daemon=True) for _ in range(workers)]
for thread in threads:
thread.start()
dropped = 0
for index in range(count):
due = starts + index / rate
delay = due - time.monotonic()
if delay > 0:
time.sleep(delay)
try:
jobs.put_nowait(due)
except Full:
dropped += 1
jobs.join()
for _ in threads:
jobs.put(None)
for thread in threads:
thread.join()
elapsed = time.monotonic() - starts
error_count = sum(errors.values()) + dropped
return {
"offered_rate": rate, "scheduled_requests": count, "successful_requests": len(latencies),
"failed_requests": error_count, "error_ratio": round(error_count / count, 6),
"successful_rps": round(len(latencies) / elapsed, 3), "elapsed_seconds": round(elapsed, 3),
"successful_latency_ms": {f"p{p}": percentile(latencies, p / 100) for p in (50, 95, 99)},
"errors_by_type": dict(errors), "queue_dropped": dropped,
"client_cpu_seconds": round(time.process_time() - cpu_start, 3),
}
def fixture(base_url, size, fixture_file):
if fixture_file.exists():
saved = json.loads(fixture_file.read_text())
if saved["base_url"] != base_url or saved["dataset_messages"] != size:
raise ValueError("saved fixture URL or size differs from requested profile")
Client(base_url).call("GET", saved["path"], actor={"token": saved["token"]})
return saved["token"], saved["path"]
client = Client(base_url)
alice, _, _, chat = client.bootstrap()
for index in range(size):
client.message(alice, chat, f"synthetic-scale-{index}")
path = "/api/v1/chats/" + chat["id"] + "/messages?limit=20"
fixture_file.parent.mkdir(parents=True, exist_ok=True)
with os.fdopen(os.open(fixture_file, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600), "w") as file:
file.write(json.dumps({"base_url": base_url, "dataset_messages": size,
"token": alice["token"], "path": path}) + "\n")
return alice["token"], path
def run(args):
full_matrix = args.modes == ["fixed", "growth"] and args.replicas == [1, 2, 4]
if args.repeats != 3 or (full_matrix and len(args.rates) < 3) or args.dataset_messages < 20:
raise ValueError("require 3 repeats, at least 3 rates for a full matrix, and at least 20 fixture messages")
args.output.parent.mkdir(parents=True, exist_ok=True)
if args.output.exists():
raise ValueError(f"output already exists: {args.output}")
token = path = None
rows = []
with tempfile.TemporaryDirectory() as temporary:
override = Path(temporary) / "scale.yaml"
for mode in args.modes:
for replicas in args.replicas:
_, cpu, memory = topology(mode, replicas)
override.write_text("services:\n api:\n cpus: " + str(cpu) + "\n mem_limit: " + str(memory) + "m\n")
compose("up", "-d", "--build", "--wait", "--scale", f"api={replicas}", override=override)
if token is None:
token, path = fixture(args.base_url, args.dataset_messages, args.fixture)
ids = docker_ids(override)
for rate in args.rates:
for repeat in range(1, args.repeats + 1):
open_loop(args.base_url, path, token, rate, args.warmup)
pool_before = pool_snapshot(args.base_url, replicas)
db_before = db_snapshot(override)
samples = []
sampling_errors = []
stop = threading.Event()
def sample():
while not stop.is_set():
try:
samples.append(stats_sample(ids))
except (subprocess.CalledProcessError, ValueError, KeyError) as error:
sampling_errors.append(type(error).__name__)
return
stop.wait(1)
sampler = threading.Thread(target=sample, daemon=True)
sampler.start()
result = open_loop(args.base_url, path, token, rate, args.duration)
stop.set()
sampler.join()
if sampling_errors or not samples:
raise RuntimeError(f"Docker CPU/RAM sampler failed: {sampling_errors}")
db_after = db_snapshot(override)
if db_after["messages"] != db_before["messages"]:
raise RuntimeError("message dataset changed during read-only benchmark")
pool_after = pool_snapshot(args.base_url, replicas)
pool_delta = {
"observed_replicas": len(set(pool_before) & set(pool_after)),
"requests_queued": sum(pool_after[i]["queued"] - pool_before[i]["queued"] for i in pool_before.keys() & pool_after.keys()),
"wait_ms": sum(pool_after[i]["wait_ms"] - pool_before[i]["wait_ms"] for i in pool_before.keys() & pool_after.keys()),
"max_size": max((v["size"] for v in pool_after.values()), default=0),
}
services = {name: {
"avg_cpu_percent": round(statistics.mean(s[name]["cpu_percent"] for s in samples if name in s), 3),
"max_memory_mib": round(max(s[name]["memory_mib"] for s in samples if name in s), 3),
} for name in ("api", "postgres", "lb") if any(name in s for s in samples)}
row = {
"mode": mode, "replicas": replicas, "rate": rate, "repeat": repeat,
"duration": args.duration, "warmup": args.warmup, "dataset_messages": args.dataset_messages,
"api_cpu_each": cpu, "api_memory_mib_each": memory, "pool_max_size": 8,
**result, "docker": services, "pool": pool_delta,
"postgres_delta": {key: db_after[key] - db_before[key] for key in db_before if key != "database_bytes"},
"database_bytes": db_after["database_bytes"], "database_messages": db_after["messages"],
}
rows.append(row)
with args.output.open("a") as output:
output.write(json.dumps(row, ensure_ascii=False) + "\n")
print(f"{mode} replicas={replicas} rate={rate} repeat={repeat} rps={result['successful_rps']} errors={result['error_ratio']}", flush=True)
if full_matrix:
validate_matrix(rows)
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--base-url", default="http://127.0.0.1:8080")
parser.add_argument("--rates", type=lambda value: [int(x) for x in value.split(",")], default=[100, 250, 500])
parser.add_argument("--modes", type=lambda value: value.split(","), default=["fixed", "growth"])
parser.add_argument("--replicas", type=lambda value: [int(x) for x in value.split(",")], default=[1, 2, 4])
parser.add_argument("--duration", type=int, default=8)
parser.add_argument("--warmup", type=int, default=2)
parser.add_argument("--repeats", type=int, default=3)
parser.add_argument("--dataset-messages", type=int, default=1000)
parser.add_argument("--fixture", type=Path, default=ROOT / "secrets/scale_fixture.json")
parser.add_argument("--output", type=Path, default=ROOT / "evidence/scale/baseline.jsonl")
args = parser.parse_args()
if (not args.rates or min(args.rates) <= 0 or not args.modes or not args.replicas or
any(mode not in ("fixed", "growth") for mode in args.modes) or
any(replica not in (1, 2, 4) for replica in args.replicas) or
args.duration <= 0 or args.warmup <= 0):
raise ValueError("rates and durations must be positive")
run(args)
if __name__ == "__main__":
main()