#!/usr/bin/env python3 """Repeatable closed-loop read benchmark for the lab 1 in-memory history.""" import argparse import base64 import csv import http.client import json import math import os from concurrent.futures import ThreadPoolExecutor from pathlib import Path import subprocess import threading import time from urllib.parse import quote PORT = 18080 NAME = "mtusi-lab1-benchmark" HISTORY = (100, 2000, 10000) CLIENTS = (1, 4, 12) FIELDS = ( "phase", "image", "history", "clients", "repeat", "seed", "duration_s", "warmup_s", "cpu_limit", "memory_limit_mib", "success", "errors", "error_ratio", "rps", "p50_ms", "p95_ms", "p99_ms", "cpu_percent_one_core", "rss_start_kib", "rss_end_kib", ) def docker(*args): return subprocess.check_output(("docker", *args), text=True).strip() class HTTP: def __init__(self): self.conn = http.client.HTTPConnection("127.0.0.1", PORT, timeout=10) def call(self, method, path, user=None, body=None): headers = {"X-User-Id": user} if user else {} if body is not None: headers["Content-Type"] = "application/json" body = json.dumps(body, separators=(",", ":")).encode() self.conn.request(method, path, body, headers) response = self.conn.getresponse() data = response.read() if response.status not in (200, 201): raise RuntimeError(f"HTTP {response.status}: {data[:120]!r}") return json.loads(data) def close(self): self.conn.close() def seed_history(count, seed): client = HTTP() try: user = client.call("POST", "/api/v1/users", body={"username": "bench", "display_name": "Benchmark"})["id"] chat = client.call("POST", "/api/v1/chats", user, {"title": "Benchmark", "member_ids": [user]})["id"] finally: client.close() def batch(worker): local = HTTP() try: return [local.call("POST", f"/api/v1/chats/{chat}/messages", user, {"text": f"synthetic-{seed}-{index:05d}"}) for index in range(worker, count, 8)] finally: local.close() with ThreadPoolExecutor(max_workers=8) as pool: messages = [message for batch_result in pool.map(batch, range(8)) for message in batch_result] messages.sort(key=lambda item: (item["created_at"], item["id"])) midpoint = messages[count // 2] cursor = base64.urlsafe_b64encode(json.dumps( [f"messages:{chat}", midpoint["created_at"], midpoint["id"]], separators=(",", ":") ).encode()).decode().rstrip("=") path = f"/api/v1/chats/{chat}/messages?limit=20&cursor={quote(cursor)}" client = HTTP() try: page = client.call("GET", path, user) assert len(page["items"]) == 20 assert (page["items"][0]["created_at"], page["items"][0]["id"]) > (midpoint["created_at"], midpoint["id"]) finally: client.close() return user, path def process_metrics(): stat = docker("exec", NAME, "cat", "/proc/1/stat").split() status = docker("exec", NAME, "cat", "/proc/1/status") rss = int(next(line.split()[1] for line in status.splitlines() if line.startswith("VmRSS:"))) return (int(stat[13]) + int(stat[14])) / os.sysconf("SC_CLK_TCK"), rss def percentile(values, fraction): return round(values[math.ceil(len(values) * fraction) - 1], 3) if values else "" def load(user, path, clients, duration): barrier = threading.Barrier(clients) deadline = time.monotonic() + duration def worker(): connection = HTTP() latencies, errors = [], 0 barrier.wait() try: while time.monotonic() < deadline: start = time.perf_counter() try: connection.call("GET", path, user) latencies.append((time.perf_counter() - start) * 1000) except (OSError, RuntimeError, ValueError): errors += 1 connection.close() connection = HTTP() finally: connection.close() return latencies, errors start = time.monotonic() with ThreadPoolExecutor(max_workers=clients) as pool: results = list(pool.map(lambda _: worker(), range(clients))) elapsed = time.monotonic() - start latencies = sorted(value for successes, _ in results for value in successes) errors = sum(failures for _, failures in results) return elapsed, latencies, errors def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--phase", choices=("baseline", "after"), required=True) parser.add_argument("--image", required=True) parser.add_argument("--output", default="evidence/load-lab1.csv") parser.add_argument("--duration", type=float, default=3) parser.add_argument("--warmup", type=float, default=1) parser.add_argument("--repeats", type=int, default=2) parser.add_argument("--seed", type=int, default=20261001) args = parser.parse_args() if args.duration <= 0 or args.warmup <= 0 or args.repeats < 2: parser.error("duration and warmup must be positive; repeats must be >= 2") output = Path(args.output) output.parent.mkdir(parents=True, exist_ok=True) new_file = not output.exists() with output.open("a", newline="") as result: writer = csv.DictWriter(result, fieldnames=FIELDS) if new_file: writer.writeheader() for history in HISTORY: docker("run", "-d", "--rm", "--name", NAME, "--cpus=1", "--memory=512m", "--memory-swap=512m", "-p", f"127.0.0.1:{PORT}:8080", args.image) try: for _ in range(100): try: HTTP().call("GET", "/health/ready") break except OSError: time.sleep(.1) else: raise RuntimeError("container did not become ready") user, path = seed_history(history, args.seed) for clients in CLIENTS: for repeat in range(1, args.repeats + 1): load(user, path, clients, args.warmup) cpu_before, rss_before = process_metrics() elapsed, latencies, errors = load(user, path, clients, args.duration) cpu_after, rss_after = process_metrics() total = len(latencies) + errors row = dict(phase=args.phase, image=args.image, history=history, clients=clients, repeat=repeat, seed=args.seed, duration_s=args.duration, warmup_s=args.warmup, cpu_limit=1, memory_limit_mib=512, success=len(latencies), errors=errors, error_ratio=round(errors / total, 6) if total else 0, rps=round(len(latencies) / elapsed, 2), p50_ms=percentile(latencies, .5), p95_ms=percentile(latencies, .95), p99_ms=percentile(latencies, .99), cpu_percent_one_core=round((cpu_after - cpu_before) / elapsed * 100, 1), rss_start_kib=rss_before, rss_end_kib=rss_after) writer.writerow(row) result.flush() print(row, flush=True) finally: docker("stop", "--time", "10", NAME) if __name__ == "__main__": main()