268 lines
9.2 KiB
Python
268 lines
9.2 KiB
Python
import argparse
|
|
import concurrent.futures
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import signal
|
|
import socket
|
|
import subprocess
|
|
import tempfile
|
|
import threading
|
|
import time
|
|
import urllib.error
|
|
import urllib.request
|
|
import uuid
|
|
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
FAST_PATH = """\tif len(history) == 0 || compareKeyset(
|
|
\t\thistory[len(history)-1].CreatedAt,
|
|
\t\thistory[len(history)-1].ID,
|
|
\t\tmessage.CreatedAt,
|
|
\t\tmessage.ID,
|
|
\t) < 0 {
|
|
\t\ts.messagesByChat[message.ChatID] = append(history, message)
|
|
|
|
\t\treturn message, nil
|
|
\t}
|
|
|
|
"""
|
|
|
|
|
|
def request(opener, base, method, path, body=None, user=None):
|
|
data = json.dumps(body, ensure_ascii=False).encode() if body is not None else None
|
|
headers = {}
|
|
if data is not None:
|
|
headers["Content-Type"] = "application/json"
|
|
if user is not None:
|
|
headers["X-User-Id"] = user
|
|
req = urllib.request.Request(base + path, data=data, headers=headers, method=method)
|
|
with opener.open(req, timeout=10) as response:
|
|
return json.loads(response.read())
|
|
|
|
|
|
def available_port():
|
|
with socket.socket() as listener:
|
|
listener.bind(("127.0.0.1", 0))
|
|
return listener.getsockname()[1]
|
|
|
|
|
|
def descendants(root_pid):
|
|
result = subprocess.run(
|
|
["ps", "-axo", "pid=,ppid=,rss=,command="],
|
|
capture_output=True,
|
|
text=True,
|
|
check=True,
|
|
)
|
|
entries = []
|
|
for line in result.stdout.splitlines():
|
|
fields = line.strip().split(None, 3)
|
|
if len(fields) == 4:
|
|
try:
|
|
entries.append((int(fields[0]), int(fields[1]), int(fields[2]), fields[3]))
|
|
except ValueError:
|
|
continue
|
|
children = {root_pid}
|
|
while True:
|
|
discovered = children | {pid for pid, ppid, _, _ in entries if ppid in children}
|
|
if discovered == children:
|
|
break
|
|
children = discovered
|
|
return [entry for entry in entries if entry[0] in children and entry[0] != root_pid]
|
|
|
|
|
|
def server_rss_kib(go_run_pid):
|
|
servers = [rss for _, _, rss, command in descendants(go_run_pid) if command.rstrip().endswith("/messenger")]
|
|
if not servers:
|
|
raise RuntimeError("go run server process was not found")
|
|
return max(servers)
|
|
|
|
|
|
def wait_ready(process, opener, base):
|
|
deadline = time.monotonic() + 60
|
|
while time.monotonic() < deadline:
|
|
if process.poll() is not None:
|
|
raise RuntimeError(f"go run exited with code {process.returncode}")
|
|
try:
|
|
request(opener, base, "GET", "/health/live")
|
|
server_rss_kib(process.pid)
|
|
return
|
|
except (OSError, RuntimeError):
|
|
time.sleep(0.1)
|
|
raise RuntimeError("go run server did not become ready")
|
|
|
|
|
|
def percentile(latencies, fraction):
|
|
ordered = sorted(latencies)
|
|
index = max(0, min(len(ordered) - 1, (len(ordered) * int(fraction * 100) + 99) // 100 - 1))
|
|
return round(ordered[index] * 1000, 3)
|
|
|
|
|
|
def setup(opener, base, history):
|
|
users = []
|
|
for _ in range(10):
|
|
user = request(opener, base, "POST", "/api/v1/users", {
|
|
"username": "b" + uuid.uuid4().hex[:20],
|
|
"display_name": "Benchmark",
|
|
})
|
|
users.append(user)
|
|
|
|
chat = request(opener, base, "POST", "/api/v1/chats", {
|
|
"title": "Benchmark",
|
|
"member_ids": [user["id"] for user in users],
|
|
}, users[0]["id"])
|
|
|
|
for index in range(history):
|
|
request(opener, base, "POST", f"/api/v1/chats/{chat['id']}/messages", {
|
|
"text": f"history-{index}",
|
|
}, users[index % len(users)]["id"])
|
|
|
|
return chat, users
|
|
|
|
|
|
def launch(mode, port, overlay):
|
|
command = ["go", "run"]
|
|
if mode == "baseline":
|
|
command.extend(["-overlay", str(overlay)])
|
|
command.append("./cmd/messenger")
|
|
process = subprocess.Popen(
|
|
command,
|
|
cwd=ROOT,
|
|
env={**os.environ, "APP_HOST": "127.0.0.1", "APP_PORT": str(port)},
|
|
stdout=subprocess.DEVNULL,
|
|
stderr=subprocess.PIPE,
|
|
start_new_session=True,
|
|
)
|
|
return process, command
|
|
|
|
|
|
def run_case(mode, history, concurrency, requests, overlay):
|
|
port = available_port()
|
|
base = f"http://127.0.0.1:{port}"
|
|
opener = urllib.request.build_opener(urllib.request.ProxyHandler({}))
|
|
process, command = launch(mode, port, overlay)
|
|
try:
|
|
wait_ready(process, opener, base)
|
|
chat, users = setup(opener, base, history)
|
|
rss = []
|
|
stop_sampling = threading.Event()
|
|
sampling_errors = []
|
|
|
|
def sample():
|
|
while not stop_sampling.is_set():
|
|
try:
|
|
rss.append(server_rss_kib(process.pid))
|
|
except RuntimeError as exc:
|
|
sampling_errors.append(str(exc))
|
|
stop_sampling.wait(0.1)
|
|
|
|
sampler = threading.Thread(target=sample)
|
|
sampler.start()
|
|
warmup = min(100, max(10, requests // 100))
|
|
|
|
for index in range(warmup):
|
|
request(opener, base, "POST", f"/api/v1/chats/{chat['id']}/messages", {
|
|
"text": f"warmup-{index}",
|
|
}, users[index % len(users)]["id"])
|
|
|
|
started = time.monotonic()
|
|
|
|
def worker(worker_id):
|
|
latencies = []
|
|
failures = 0
|
|
worker_requests = requests // concurrency + (1 if worker_id < requests % concurrency else 0)
|
|
for index in range(worker_requests):
|
|
request_started = time.perf_counter()
|
|
try:
|
|
request(opener, base, "POST", f"/api/v1/chats/{chat['id']}/messages", {
|
|
"text": f"load-{worker_id}-{index}",
|
|
}, users[worker_id % len(users)]["id"])
|
|
latencies.append(time.perf_counter() - request_started)
|
|
except (OSError, ValueError, urllib.error.HTTPError):
|
|
failures += 1
|
|
index += 1
|
|
return latencies, failures
|
|
|
|
try:
|
|
with concurrent.futures.ThreadPoolExecutor(max_workers=concurrency) as pool:
|
|
results = list(pool.map(worker, range(concurrency)))
|
|
finally:
|
|
elapsed = time.monotonic() - started
|
|
stop_sampling.set()
|
|
sampler.join()
|
|
|
|
latencies = [value for values, _ in results for value in values]
|
|
failures = sum(value for _, value in results)
|
|
if not latencies or not rss or sampling_errors:
|
|
raise RuntimeError(f"invalid sample: successes={len(latencies)} rss={len(rss)} errors={sampling_errors}")
|
|
|
|
return {
|
|
"history": history,
|
|
"concurrency": concurrency,
|
|
"duration_seconds": round(elapsed, 3),
|
|
"requested_requests": requests,
|
|
"warmup_requests": warmup,
|
|
"successful_requests": len(latencies),
|
|
"failed_requests": failures,
|
|
"rps": round(len(latencies) / elapsed, 3),
|
|
"p50_ms": percentile(latencies, 0.50),
|
|
"p95_ms": percentile(latencies, 0.95),
|
|
"p99_ms": percentile(latencies, 0.99),
|
|
"rss_min_kib": min(rss),
|
|
"rss_max_kib": max(rss),
|
|
"rss_delta_kib": max(rss) - min(rss),
|
|
}, command
|
|
finally:
|
|
os.killpg(process.pid, signal.SIGTERM)
|
|
try:
|
|
process.wait(timeout=10)
|
|
except subprocess.TimeoutExpired:
|
|
os.killpg(process.pid, signal.SIGKILL)
|
|
process.wait()
|
|
|
|
|
|
def baseline_overlay(directory):
|
|
source = (ROOT / "internal/storage/memory/message_repository.go").resolve()
|
|
content = source.read_text()
|
|
if content.count(FAST_PATH) != 1:
|
|
raise RuntimeError("expected exactly one optimized append block")
|
|
replacement = directory / "message_repository_baseline.go"
|
|
replacement.write_text(content.replace(FAST_PATH, "", 1))
|
|
overlay = directory / "overlay.json"
|
|
overlay.write_text(json.dumps({"Replace": {str(source): str(replacement)}}))
|
|
return overlay
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("--mode", choices=("baseline", "optimized"), required=True)
|
|
parser.add_argument("--requests", type=int, default=10000)
|
|
parser.add_argument("--output", type=Path, required=True)
|
|
args = parser.parse_args()
|
|
if args.requests <= 0:
|
|
parser.error("--requests must be positive")
|
|
|
|
with tempfile.TemporaryDirectory(prefix="messenger-track1-") as temp_dir:
|
|
overlay = baseline_overlay(Path(temp_dir))
|
|
rows = []
|
|
for history in (1000, 10000, 50000):
|
|
for concurrency in (1, 4, 16, 32):
|
|
row, command = run_case(args.mode, history, concurrency, args.requests, overlay)
|
|
rows.append(row)
|
|
print(json.dumps(row, ensure_ascii=False), flush=True)
|
|
result = {
|
|
"mode": args.mode,
|
|
"command": command,
|
|
"go_version": subprocess.check_output(["go", "version"], text=True).strip(),
|
|
"os_arch": subprocess.check_output(["go", "env", "GOOS", "GOARCH"], text=True).split(),
|
|
"requests_per_case": args.requests,
|
|
"rss_source": "ps RSS of go run child executable, KiB; sampled every 100 ms",
|
|
"rows": rows,
|
|
}
|
|
args.output.parent.mkdir(parents=True, exist_ok=True)
|
|
args.output.write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|