Files
mtusi2026lab2/scripts/track1_benchmark.py
T
2026-10-01 12:59:06 +03:00

268 lines
9.2 KiB
Python

import argparse
import concurrent.futures
import json
import os
from pathlib import Path
import signal
import socket
import subprocess
import tempfile
import threading
import time
import urllib.error
import urllib.request
import uuid
ROOT = Path(__file__).resolve().parents[1]
FAST_PATH = """\tif len(history) == 0 || compareKeyset(
\t\thistory[len(history)-1].CreatedAt,
\t\thistory[len(history)-1].ID,
\t\tmessage.CreatedAt,
\t\tmessage.ID,
\t) < 0 {
\t\ts.messagesByChat[message.ChatID] = append(history, message)
\t\treturn message, nil
\t}
"""
def request(opener, base, method, path, body=None, user=None):
data = json.dumps(body, ensure_ascii=False).encode() if body is not None else None
headers = {}
if data is not None:
headers["Content-Type"] = "application/json"
if user is not None:
headers["X-User-Id"] = user
req = urllib.request.Request(base + path, data=data, headers=headers, method=method)
with opener.open(req, timeout=10) as response:
return json.loads(response.read())
def available_port():
with socket.socket() as listener:
listener.bind(("127.0.0.1", 0))
return listener.getsockname()[1]
def descendants(root_pid):
result = subprocess.run(
["ps", "-axo", "pid=,ppid=,rss=,command="],
capture_output=True,
text=True,
check=True,
)
entries = []
for line in result.stdout.splitlines():
fields = line.strip().split(None, 3)
if len(fields) == 4:
try:
entries.append((int(fields[0]), int(fields[1]), int(fields[2]), fields[3]))
except ValueError:
continue
children = {root_pid}
while True:
discovered = children | {pid for pid, ppid, _, _ in entries if ppid in children}
if discovered == children:
break
children = discovered
return [entry for entry in entries if entry[0] in children and entry[0] != root_pid]
def server_rss_kib(go_run_pid):
servers = [rss for _, _, rss, command in descendants(go_run_pid) if command.rstrip().endswith("/messenger")]
if not servers:
raise RuntimeError("go run server process was not found")
return max(servers)
def wait_ready(process, opener, base):
deadline = time.monotonic() + 60
while time.monotonic() < deadline:
if process.poll() is not None:
raise RuntimeError(f"go run exited with code {process.returncode}")
try:
request(opener, base, "GET", "/health/live")
server_rss_kib(process.pid)
return
except (OSError, RuntimeError):
time.sleep(0.1)
raise RuntimeError("go run server did not become ready")
def percentile(latencies, fraction):
ordered = sorted(latencies)
index = max(0, min(len(ordered) - 1, (len(ordered) * int(fraction * 100) + 99) // 100 - 1))
return round(ordered[index] * 1000, 3)
def setup(opener, base, history):
users = []
for _ in range(10):
user = request(opener, base, "POST", "/api/v1/users", {
"username": "b" + uuid.uuid4().hex[:20],
"display_name": "Benchmark",
})
users.append(user)
chat = request(opener, base, "POST", "/api/v1/chats", {
"title": "Benchmark",
"member_ids": [user["id"] for user in users],
}, users[0]["id"])
for index in range(history):
request(opener, base, "POST", f"/api/v1/chats/{chat['id']}/messages", {
"text": f"history-{index}",
}, users[index % len(users)]["id"])
return chat, users
def launch(mode, port, overlay):
command = ["go", "run"]
if mode == "baseline":
command.extend(["-overlay", str(overlay)])
command.append("./cmd/messenger")
process = subprocess.Popen(
command,
cwd=ROOT,
env={**os.environ, "APP_HOST": "127.0.0.1", "APP_PORT": str(port)},
stdout=subprocess.DEVNULL,
stderr=subprocess.PIPE,
start_new_session=True,
)
return process, command
def run_case(mode, history, concurrency, requests, overlay):
port = available_port()
base = f"http://127.0.0.1:{port}"
opener = urllib.request.build_opener(urllib.request.ProxyHandler({}))
process, command = launch(mode, port, overlay)
try:
wait_ready(process, opener, base)
chat, users = setup(opener, base, history)
rss = []
stop_sampling = threading.Event()
sampling_errors = []
def sample():
while not stop_sampling.is_set():
try:
rss.append(server_rss_kib(process.pid))
except RuntimeError as exc:
sampling_errors.append(str(exc))
stop_sampling.wait(0.1)
sampler = threading.Thread(target=sample)
sampler.start()
warmup = min(100, max(10, requests // 100))
for index in range(warmup):
request(opener, base, "POST", f"/api/v1/chats/{chat['id']}/messages", {
"text": f"warmup-{index}",
}, users[index % len(users)]["id"])
started = time.monotonic()
def worker(worker_id):
latencies = []
failures = 0
worker_requests = requests // concurrency + (1 if worker_id < requests % concurrency else 0)
for index in range(worker_requests):
request_started = time.perf_counter()
try:
request(opener, base, "POST", f"/api/v1/chats/{chat['id']}/messages", {
"text": f"load-{worker_id}-{index}",
}, users[worker_id % len(users)]["id"])
latencies.append(time.perf_counter() - request_started)
except (OSError, ValueError, urllib.error.HTTPError):
failures += 1
index += 1
return latencies, failures
try:
with concurrent.futures.ThreadPoolExecutor(max_workers=concurrency) as pool:
results = list(pool.map(worker, range(concurrency)))
finally:
elapsed = time.monotonic() - started
stop_sampling.set()
sampler.join()
latencies = [value for values, _ in results for value in values]
failures = sum(value for _, value in results)
if not latencies or not rss or sampling_errors:
raise RuntimeError(f"invalid sample: successes={len(latencies)} rss={len(rss)} errors={sampling_errors}")
return {
"history": history,
"concurrency": concurrency,
"duration_seconds": round(elapsed, 3),
"requested_requests": requests,
"warmup_requests": warmup,
"successful_requests": len(latencies),
"failed_requests": failures,
"rps": round(len(latencies) / elapsed, 3),
"p50_ms": percentile(latencies, 0.50),
"p95_ms": percentile(latencies, 0.95),
"p99_ms": percentile(latencies, 0.99),
"rss_min_kib": min(rss),
"rss_max_kib": max(rss),
"rss_delta_kib": max(rss) - min(rss),
}, command
finally:
os.killpg(process.pid, signal.SIGTERM)
try:
process.wait(timeout=10)
except subprocess.TimeoutExpired:
os.killpg(process.pid, signal.SIGKILL)
process.wait()
def baseline_overlay(directory):
source = (ROOT / "internal/storage/memory/message_repository.go").resolve()
content = source.read_text()
if content.count(FAST_PATH) != 1:
raise RuntimeError("expected exactly one optimized append block")
replacement = directory / "message_repository_baseline.go"
replacement.write_text(content.replace(FAST_PATH, "", 1))
overlay = directory / "overlay.json"
overlay.write_text(json.dumps({"Replace": {str(source): str(replacement)}}))
return overlay
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--mode", choices=("baseline", "optimized"), required=True)
parser.add_argument("--requests", type=int, default=10000)
parser.add_argument("--output", type=Path, required=True)
args = parser.parse_args()
if args.requests <= 0:
parser.error("--requests must be positive")
with tempfile.TemporaryDirectory(prefix="messenger-track1-") as temp_dir:
overlay = baseline_overlay(Path(temp_dir))
rows = []
for history in (1000, 10000, 50000):
for concurrency in (1, 4, 16, 32):
row, command = run_case(args.mode, history, concurrency, args.requests, overlay)
rows.append(row)
print(json.dumps(row, ensure_ascii=False), flush=True)
result = {
"mode": args.mode,
"command": command,
"go_version": subprocess.check_output(["go", "version"], text=True).strip(),
"os_arch": subprocess.check_output(["go", "env", "GOOS", "GOARCH"], text=True).split(),
"requests_per_case": args.requests,
"rss_source": "ps RSS of go run child executable, KiB; sampled every 100 ms",
"rows": rows,
}
args.output.parent.mkdir(parents=True, exist_ok=True)
args.output.write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n")
if __name__ == "__main__":
main()