Files
mtusi2026lab1/student-tests/benchmark_memory.py
T

188 lines
7.5 KiB
Python

#!/usr/bin/env python3
"""Repeatable closed-loop read benchmark for the lab 1 in-memory history."""
import argparse
import base64
import csv
import http.client
import json
import math
import os
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
import subprocess
import threading
import time
from urllib.parse import quote
PORT = 18080
NAME = "mtusi-lab1-benchmark"
HISTORY = (100, 2000, 10000)
CLIENTS = (1, 4, 12)
FIELDS = (
"phase", "image", "history", "clients", "repeat", "seed", "duration_s", "warmup_s",
"cpu_limit", "memory_limit_mib", "success", "errors", "error_ratio", "rps",
"p50_ms", "p95_ms", "p99_ms", "cpu_percent_one_core", "rss_start_kib", "rss_end_kib",
)
def docker(*args):
return subprocess.check_output(("docker", *args), text=True).strip()
class HTTP:
def __init__(self):
self.conn = http.client.HTTPConnection("127.0.0.1", PORT, timeout=10)
def call(self, method, path, user=None, body=None):
headers = {"X-User-Id": user} if user else {}
if body is not None:
headers["Content-Type"] = "application/json"
body = json.dumps(body, separators=(",", ":")).encode()
self.conn.request(method, path, body, headers)
response = self.conn.getresponse()
data = response.read()
if response.status not in (200, 201):
raise RuntimeError(f"HTTP {response.status}: {data[:120]!r}")
return json.loads(data)
def close(self):
self.conn.close()
def seed_history(count, seed):
client = HTTP()
try:
user = client.call("POST", "/api/v1/users", body={"username": "bench", "display_name": "Benchmark"})["id"]
chat = client.call("POST", "/api/v1/chats", user, {"title": "Benchmark", "member_ids": [user]})["id"]
finally:
client.close()
def batch(worker):
local = HTTP()
try:
return [local.call("POST", f"/api/v1/chats/{chat}/messages", user,
{"text": f"synthetic-{seed}-{index:05d}"})
for index in range(worker, count, 8)]
finally:
local.close()
with ThreadPoolExecutor(max_workers=8) as pool:
messages = [message for batch_result in pool.map(batch, range(8)) for message in batch_result]
messages.sort(key=lambda item: (item["created_at"], item["id"]))
midpoint = messages[count // 2]
cursor = base64.urlsafe_b64encode(json.dumps(
[f"messages:{chat}", midpoint["created_at"], midpoint["id"]], separators=(",", ":")
).encode()).decode().rstrip("=")
path = f"/api/v1/chats/{chat}/messages?limit=20&cursor={quote(cursor)}"
client = HTTP()
try:
page = client.call("GET", path, user)
assert len(page["items"]) == 20
assert (page["items"][0]["created_at"], page["items"][0]["id"]) > (midpoint["created_at"], midpoint["id"])
finally:
client.close()
return user, path
def process_metrics():
stat = docker("exec", NAME, "cat", "/proc/1/stat").split()
status = docker("exec", NAME, "cat", "/proc/1/status")
rss = int(next(line.split()[1] for line in status.splitlines() if line.startswith("VmRSS:")))
return (int(stat[13]) + int(stat[14])) / os.sysconf("SC_CLK_TCK"), rss
def percentile(values, fraction):
return round(values[math.ceil(len(values) * fraction) - 1], 3) if values else ""
def load(user, path, clients, duration):
barrier = threading.Barrier(clients)
deadline = time.monotonic() + duration
def worker():
connection = HTTP()
latencies, errors = [], 0
barrier.wait()
try:
while time.monotonic() < deadline:
start = time.perf_counter()
try:
connection.call("GET", path, user)
latencies.append((time.perf_counter() - start) * 1000)
except (OSError, RuntimeError, ValueError):
errors += 1
connection.close()
connection = HTTP()
finally:
connection.close()
return latencies, errors
start = time.monotonic()
with ThreadPoolExecutor(max_workers=clients) as pool:
results = list(pool.map(lambda _: worker(), range(clients)))
elapsed = time.monotonic() - start
latencies = sorted(value for successes, _ in results for value in successes)
errors = sum(failures for _, failures in results)
return elapsed, latencies, errors
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--phase", choices=("baseline", "after"), required=True)
parser.add_argument("--image", required=True)
parser.add_argument("--output", default="evidence/load-lab1.csv")
parser.add_argument("--duration", type=float, default=3)
parser.add_argument("--warmup", type=float, default=1)
parser.add_argument("--repeats", type=int, default=2)
parser.add_argument("--seed", type=int, default=20261001)
args = parser.parse_args()
if args.duration <= 0 or args.warmup <= 0 or args.repeats < 2:
parser.error("duration and warmup must be positive; repeats must be >= 2")
output = Path(args.output)
output.parent.mkdir(parents=True, exist_ok=True)
new_file = not output.exists()
with output.open("a", newline="") as result:
writer = csv.DictWriter(result, fieldnames=FIELDS)
if new_file:
writer.writeheader()
for history in HISTORY:
docker("run", "-d", "--rm", "--name", NAME, "--cpus=1", "--memory=512m",
"--memory-swap=512m", "-p", f"127.0.0.1:{PORT}:8080", args.image)
try:
for _ in range(100):
try:
HTTP().call("GET", "/health/ready")
break
except OSError:
time.sleep(.1)
else:
raise RuntimeError("container did not become ready")
user, path = seed_history(history, args.seed)
for clients in CLIENTS:
for repeat in range(1, args.repeats + 1):
load(user, path, clients, args.warmup)
cpu_before, rss_before = process_metrics()
elapsed, latencies, errors = load(user, path, clients, args.duration)
cpu_after, rss_after = process_metrics()
total = len(latencies) + errors
row = dict(phase=args.phase, image=args.image, history=history, clients=clients,
repeat=repeat, seed=args.seed, duration_s=args.duration, warmup_s=args.warmup,
cpu_limit=1, memory_limit_mib=512, success=len(latencies), errors=errors,
error_ratio=round(errors / total, 6) if total else 0,
rps=round(len(latencies) / elapsed, 2),
p50_ms=percentile(latencies, .5), p95_ms=percentile(latencies, .95),
p99_ms=percentile(latencies, .99),
cpu_percent_one_core=round((cpu_after - cpu_before) / elapsed * 100, 1),
rss_start_kib=rss_before, rss_end_kib=rss_after)
writer.writerow(row)
result.flush()
print(row, flush=True)
finally:
docker("stop", "--time", "10", NAME)
if __name__ == "__main__":
main()