188 lines
7.5 KiB
Python
188 lines
7.5 KiB
Python
#!/usr/bin/env python3
|
|
"""Repeatable closed-loop read benchmark for the lab 1 in-memory history."""
|
|
|
|
import argparse
|
|
import base64
|
|
import csv
|
|
import http.client
|
|
import json
|
|
import math
|
|
import os
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
from pathlib import Path
|
|
import subprocess
|
|
import threading
|
|
import time
|
|
from urllib.parse import quote
|
|
|
|
|
|
PORT = 18080
|
|
NAME = "mtusi-lab1-benchmark"
|
|
HISTORY = (100, 2000, 10000)
|
|
CLIENTS = (1, 4, 12)
|
|
FIELDS = (
|
|
"phase", "image", "history", "clients", "repeat", "seed", "duration_s", "warmup_s",
|
|
"cpu_limit", "memory_limit_mib", "success", "errors", "error_ratio", "rps",
|
|
"p50_ms", "p95_ms", "p99_ms", "cpu_percent_one_core", "rss_start_kib", "rss_end_kib",
|
|
)
|
|
|
|
|
|
def docker(*args):
|
|
return subprocess.check_output(("docker", *args), text=True).strip()
|
|
|
|
|
|
class HTTP:
|
|
def __init__(self):
|
|
self.conn = http.client.HTTPConnection("127.0.0.1", PORT, timeout=10)
|
|
|
|
def call(self, method, path, user=None, body=None):
|
|
headers = {"X-User-Id": user} if user else {}
|
|
if body is not None:
|
|
headers["Content-Type"] = "application/json"
|
|
body = json.dumps(body, separators=(",", ":")).encode()
|
|
self.conn.request(method, path, body, headers)
|
|
response = self.conn.getresponse()
|
|
data = response.read()
|
|
if response.status not in (200, 201):
|
|
raise RuntimeError(f"HTTP {response.status}: {data[:120]!r}")
|
|
return json.loads(data)
|
|
|
|
def close(self):
|
|
self.conn.close()
|
|
|
|
|
|
def seed_history(count, seed):
|
|
client = HTTP()
|
|
try:
|
|
user = client.call("POST", "/api/v1/users", body={"username": "bench", "display_name": "Benchmark"})["id"]
|
|
chat = client.call("POST", "/api/v1/chats", user, {"title": "Benchmark", "member_ids": [user]})["id"]
|
|
finally:
|
|
client.close()
|
|
|
|
def batch(worker):
|
|
local = HTTP()
|
|
try:
|
|
return [local.call("POST", f"/api/v1/chats/{chat}/messages", user,
|
|
{"text": f"synthetic-{seed}-{index:05d}"})
|
|
for index in range(worker, count, 8)]
|
|
finally:
|
|
local.close()
|
|
|
|
with ThreadPoolExecutor(max_workers=8) as pool:
|
|
messages = [message for batch_result in pool.map(batch, range(8)) for message in batch_result]
|
|
messages.sort(key=lambda item: (item["created_at"], item["id"]))
|
|
midpoint = messages[count // 2]
|
|
cursor = base64.urlsafe_b64encode(json.dumps(
|
|
[f"messages:{chat}", midpoint["created_at"], midpoint["id"]], separators=(",", ":")
|
|
).encode()).decode().rstrip("=")
|
|
path = f"/api/v1/chats/{chat}/messages?limit=20&cursor={quote(cursor)}"
|
|
client = HTTP()
|
|
try:
|
|
page = client.call("GET", path, user)
|
|
assert len(page["items"]) == 20
|
|
assert (page["items"][0]["created_at"], page["items"][0]["id"]) > (midpoint["created_at"], midpoint["id"])
|
|
finally:
|
|
client.close()
|
|
return user, path
|
|
|
|
|
|
def process_metrics():
|
|
stat = docker("exec", NAME, "cat", "/proc/1/stat").split()
|
|
status = docker("exec", NAME, "cat", "/proc/1/status")
|
|
rss = int(next(line.split()[1] for line in status.splitlines() if line.startswith("VmRSS:")))
|
|
return (int(stat[13]) + int(stat[14])) / os.sysconf("SC_CLK_TCK"), rss
|
|
|
|
|
|
def percentile(values, fraction):
|
|
return round(values[math.ceil(len(values) * fraction) - 1], 3) if values else ""
|
|
|
|
|
|
def load(user, path, clients, duration):
|
|
barrier = threading.Barrier(clients)
|
|
deadline = time.monotonic() + duration
|
|
|
|
def worker():
|
|
connection = HTTP()
|
|
latencies, errors = [], 0
|
|
barrier.wait()
|
|
try:
|
|
while time.monotonic() < deadline:
|
|
start = time.perf_counter()
|
|
try:
|
|
connection.call("GET", path, user)
|
|
latencies.append((time.perf_counter() - start) * 1000)
|
|
except (OSError, RuntimeError, ValueError):
|
|
errors += 1
|
|
connection.close()
|
|
connection = HTTP()
|
|
finally:
|
|
connection.close()
|
|
return latencies, errors
|
|
|
|
start = time.monotonic()
|
|
with ThreadPoolExecutor(max_workers=clients) as pool:
|
|
results = list(pool.map(lambda _: worker(), range(clients)))
|
|
elapsed = time.monotonic() - start
|
|
latencies = sorted(value for successes, _ in results for value in successes)
|
|
errors = sum(failures for _, failures in results)
|
|
return elapsed, latencies, errors
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--phase", choices=("baseline", "after"), required=True)
|
|
parser.add_argument("--image", required=True)
|
|
parser.add_argument("--output", default="evidence/load-lab1.csv")
|
|
parser.add_argument("--duration", type=float, default=3)
|
|
parser.add_argument("--warmup", type=float, default=1)
|
|
parser.add_argument("--repeats", type=int, default=2)
|
|
parser.add_argument("--seed", type=int, default=20261001)
|
|
args = parser.parse_args()
|
|
if args.duration <= 0 or args.warmup <= 0 or args.repeats < 2:
|
|
parser.error("duration and warmup must be positive; repeats must be >= 2")
|
|
output = Path(args.output)
|
|
output.parent.mkdir(parents=True, exist_ok=True)
|
|
new_file = not output.exists()
|
|
with output.open("a", newline="") as result:
|
|
writer = csv.DictWriter(result, fieldnames=FIELDS)
|
|
if new_file:
|
|
writer.writeheader()
|
|
for history in HISTORY:
|
|
docker("run", "-d", "--rm", "--name", NAME, "--cpus=1", "--memory=512m",
|
|
"--memory-swap=512m", "-p", f"127.0.0.1:{PORT}:8080", args.image)
|
|
try:
|
|
for _ in range(100):
|
|
try:
|
|
HTTP().call("GET", "/health/ready")
|
|
break
|
|
except OSError:
|
|
time.sleep(.1)
|
|
else:
|
|
raise RuntimeError("container did not become ready")
|
|
user, path = seed_history(history, args.seed)
|
|
for clients in CLIENTS:
|
|
for repeat in range(1, args.repeats + 1):
|
|
load(user, path, clients, args.warmup)
|
|
cpu_before, rss_before = process_metrics()
|
|
elapsed, latencies, errors = load(user, path, clients, args.duration)
|
|
cpu_after, rss_after = process_metrics()
|
|
total = len(latencies) + errors
|
|
row = dict(phase=args.phase, image=args.image, history=history, clients=clients,
|
|
repeat=repeat, seed=args.seed, duration_s=args.duration, warmup_s=args.warmup,
|
|
cpu_limit=1, memory_limit_mib=512, success=len(latencies), errors=errors,
|
|
error_ratio=round(errors / total, 6) if total else 0,
|
|
rps=round(len(latencies) / elapsed, 2),
|
|
p50_ms=percentile(latencies, .5), p95_ms=percentile(latencies, .95),
|
|
p99_ms=percentile(latencies, .99),
|
|
cpu_percent_one_core=round((cpu_after - cpu_before) / elapsed * 100, 1),
|
|
rss_start_kib=rss_before, rss_end_kib=rss_after)
|
|
writer.writerow(row)
|
|
result.flush()
|
|
print(row, flush=True)
|
|
finally:
|
|
docker("stop", "--time", "10", NAME)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|