This commit is contained in:
Vladislav Kunetskiy
2026-10-01 12:59:06 +03:00
parent 65003b6119
commit 51b236a69f
86 changed files with 33332 additions and 16 deletions
+271
View File
@@ -0,0 +1,271 @@
import argparse
import json
import os
import subprocess
import threading
import time
import urllib.error
import urllib.request
import uuid
import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt
def request(base_url, method, path, body=None, user_id=None):
payload = json.dumps(body, ensure_ascii=False).encode() if body is not None else None
headers = {"Content-Type": "application/json"} if payload else {}
if user_id:
headers["X-User-Id"] = user_id
request = urllib.request.Request(
base_url + path,
data=payload,
headers=headers,
method=method,
)
with urllib.request.urlopen(request, timeout=3) as response:
return json.loads(response.read())
def wait_ready(base_url):
deadline = time.monotonic() + 20
while time.monotonic() < deadline:
try:
request(base_url, "GET", "/health/live")
return
except OSError:
time.sleep(0.1)
raise RuntimeError("container did not become ready")
def process_alive(container):
result = subprocess.run(
["docker", "inspect", "--format", "{{.State.Running}}", container],
capture_output=True,
text=True,
check=False,
)
return result.stdout.strip() == "true"
def sample_stats(container, samples, stop):
while not stop.is_set():
result = subprocess.run(
["docker", "stats", "--no-stream", "--format", "{{json .}}", container],
capture_output=True,
text=True,
check=False,
)
if result.returncode == 0 and result.stdout.strip():
try:
value = json.loads(result.stdout)
memory = value.get("MemUsage", "").split("/")[0].strip()
if memory == "0B" or not process_alive(container):
continue
samples.append({
"time_seconds": time.monotonic(),
"memory": memory,
"memory_mib": parse_memory(memory),
})
except (ValueError, IndexError):
pass
stop.wait(0.1)
def parse_memory(value):
units = {"B": 1 / 1024 / 1024, "KiB": 1 / 1024, "MiB": 1, "GiB": 1024}
for unit, factor in sorted(units.items(), key=lambda item: -len(item[0])):
if value.endswith(unit):
return float(value[:-len(unit)].strip()) * factor
raise ValueError(value)
def make_plot(data, output):
samples = [sample for sample in data["memory_samples"] if sample["memory_mib"] > 0]
start = samples[0]["time_seconds"] if samples else 0
x = [sample["time_seconds"] - start for sample in samples]
memory = [sample["memory_mib"] for sample in samples]
fig, memory_axis = plt.subplots(figsize=(11, 6))
memory_axis.plot(x, memory, color="tab:red", label="RSS / container memory")
memory_axis.axhline(data["memory_limit_mib"], color="black", linestyle="--", label="memory limit")
memory_axis.set_xlabel("Time, seconds")
memory_axis.set_ylabel("Memory, MiB")
memory_axis.grid(True, alpha=0.25)
message_axis = memory_axis.twinx()
message_axis.plot(
data["message_points_x"],
data["message_points_y"],
color="tab:blue",
label="successful messages",
)
message_axis.set_ylabel("Successful messages")
lines, labels = memory_axis.get_legend_handles_labels()
other_lines, other_labels = message_axis.get_legend_handles_labels()
memory_axis.legend(lines + other_lines, labels + other_labels, loc="upper left")
memory_axis.set_title("In-memory messenger: memory exhaustion point")
if data["oom_killed"] and data["failure_time_seconds"] is not None:
memory_axis.axvline(
data["failure_time_seconds"],
color="tab:purple",
linestyle=":",
label="container OOMKilled",
)
memory_axis.annotate(
"OOMKilled",
(data["failure_time_seconds"], data["memory_limit_mib"]),
xytext=(8, -18),
textcoords="offset points",
color="tab:purple",
)
fig.tight_layout()
fig.savefig(output, dpi=160)
plt.close(fig)
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--image", required=True)
parser.add_argument("--memory", default="32m")
parser.add_argument("--output", default="evidence/memory-experiment.json")
parser.add_argument("--plot", default="evidence/memory-experiment.png")
parser.add_argument("--max-messages", type=int, default=100000)
args = parser.parse_args()
container = "mtusi-memory-experiment-" + uuid.uuid4().hex[:8]
port = 18081
base_url = f"http://127.0.0.1:{port}"
subprocess.run(
[
"docker", "run", "-d", "--name", container,
"--memory", args.memory, "--memory-swap", args.memory,
"-p", f"127.0.0.1:{port}:8080", args.image,
],
check=True,
capture_output=True,
text=True,
)
samples = []
stop_sampling = threading.Event()
sampler = threading.Thread(target=sample_stats, args=(container, samples, stop_sampling))
successful = 0
failures = 0
fatal_error = None
started = time.monotonic()
message_points_x = []
message_points_y = []
failure_time = None
oom_killed = False
last_error = None
last_state = {}
logs = ""
try:
wait_ready(base_url)
user = request(base_url, "POST", "/api/v1/users", {
"username": "memory_" + uuid.uuid4().hex[:16],
"display_name": "Memory experiment",
})
chat = request(base_url, "POST", "/api/v1/chats", {
"title": "Memory experiment",
"member_ids": [user["id"]],
}, user["id"])
sampler.start()
time.sleep(0.5)
body = "x" * 4000
for index in range(args.max_messages):
try:
request(
base_url,
"POST",
f"/api/v1/chats/{chat['id']}/messages",
{"text": body},
user["id"],
)
successful += 1
message_points_x.append(time.monotonic() - started)
message_points_y.append(successful)
except (OSError, urllib.error.HTTPError, ValueError) as exc:
last_error = type(exc).__name__ + ": " + str(exc)
failures += 1
failure_time = time.monotonic() - started
break
time.sleep(0.5)
if not process_alive(container):
failure_time = failure_time or (time.monotonic() - started)
inspect = subprocess.run(
["docker", "inspect", "--format", "{{json .State}}", container],
capture_output=True,
text=True,
check=False,
)
state = json.loads(inspect.stdout) if inspect.stdout.strip() else {}
if not state:
state_result = subprocess.run(
["docker", "inspect", "--format", "{{json .State}}", container],
capture_output=True,
text=True,
check=False,
)
state = json.loads(state_result.stdout) if state_result.stdout.strip() else {}
last_state = state
logs_result = subprocess.run(["docker", "logs", container], capture_output=True, text=True, check=False)
logs = logs_result.stdout + logs_result.stderr
oom_killed = bool(state.get("OOMKilled"))
if failure_time is None and failures:
failure_time = time.monotonic() - started
except Exception as exc:
fatal_error = type(exc).__name__ + ": " + str(exc)
failure_time = failure_time or (time.monotonic() - started)
finally:
stop_sampling.set()
if sampler.is_alive():
sampler.join(timeout=2)
memory_limit_mib = float(args.memory.rstrip("mM")) if args.memory.lower().endswith("m") else 0
if not oom_killed:
inspect = subprocess.run(
["docker", "inspect", "--format", "{{json .State}}", container],
capture_output=True,
text=True,
check=False,
)
state = json.loads(inspect.stdout) if inspect.stdout.strip() else {}
last_state = state
oom_killed = bool(state.get("OOMKilled"))
result = {
"image": args.image,
"memory_limit": args.memory,
"memory_limit_mib": memory_limit_mib,
"successful_messages": successful,
"failed_requests": failures,
"oom_killed": oom_killed,
"last_error": last_error,
"fatal_error": fatal_error,
"container_state": last_state,
"container_logs_tail": logs[-20000:],
"failure_time_seconds": failure_time,
"memory_samples": samples,
"message_points_x": message_points_x,
"message_points_y": message_points_y,
"reason": "process stopped after the container memory limit was reached" if oom_killed else "request processing did not reach the memory limit",
}
with open(args.output, "w", encoding="utf-8") as output:
json.dump(result, output, ensure_ascii=False, indent=2)
make_plot(result, args.plot)
print(json.dumps({key: result[key] for key in (
"memory_limit", "successful_messages", "failed_requests", "oom_killed", "failure_time_seconds", "reason",
)}, ensure_ascii=False, indent=2))
subprocess.run(["docker", "rm", "-f", container], capture_output=True, check=False)
if __name__ == "__main__":
main()
+267
View File
@@ -0,0 +1,267 @@
import argparse
import concurrent.futures
import json
import os
from pathlib import Path
import signal
import socket
import subprocess
import tempfile
import threading
import time
import urllib.error
import urllib.request
import uuid
ROOT = Path(__file__).resolve().parents[1]
FAST_PATH = """\tif len(history) == 0 || compareKeyset(
\t\thistory[len(history)-1].CreatedAt,
\t\thistory[len(history)-1].ID,
\t\tmessage.CreatedAt,
\t\tmessage.ID,
\t) < 0 {
\t\ts.messagesByChat[message.ChatID] = append(history, message)
\t\treturn message, nil
\t}
"""
def request(opener, base, method, path, body=None, user=None):
data = json.dumps(body, ensure_ascii=False).encode() if body is not None else None
headers = {}
if data is not None:
headers["Content-Type"] = "application/json"
if user is not None:
headers["X-User-Id"] = user
req = urllib.request.Request(base + path, data=data, headers=headers, method=method)
with opener.open(req, timeout=10) as response:
return json.loads(response.read())
def available_port():
with socket.socket() as listener:
listener.bind(("127.0.0.1", 0))
return listener.getsockname()[1]
def descendants(root_pid):
result = subprocess.run(
["ps", "-axo", "pid=,ppid=,rss=,command="],
capture_output=True,
text=True,
check=True,
)
entries = []
for line in result.stdout.splitlines():
fields = line.strip().split(None, 3)
if len(fields) == 4:
try:
entries.append((int(fields[0]), int(fields[1]), int(fields[2]), fields[3]))
except ValueError:
continue
children = {root_pid}
while True:
discovered = children | {pid for pid, ppid, _, _ in entries if ppid in children}
if discovered == children:
break
children = discovered
return [entry for entry in entries if entry[0] in children and entry[0] != root_pid]
def server_rss_kib(go_run_pid):
servers = [rss for _, _, rss, command in descendants(go_run_pid) if command.rstrip().endswith("/messenger")]
if not servers:
raise RuntimeError("go run server process was not found")
return max(servers)
def wait_ready(process, opener, base):
deadline = time.monotonic() + 60
while time.monotonic() < deadline:
if process.poll() is not None:
raise RuntimeError(f"go run exited with code {process.returncode}")
try:
request(opener, base, "GET", "/health/live")
server_rss_kib(process.pid)
return
except (OSError, RuntimeError):
time.sleep(0.1)
raise RuntimeError("go run server did not become ready")
def percentile(latencies, fraction):
ordered = sorted(latencies)
index = max(0, min(len(ordered) - 1, (len(ordered) * int(fraction * 100) + 99) // 100 - 1))
return round(ordered[index] * 1000, 3)
def setup(opener, base, history):
users = []
for _ in range(10):
user = request(opener, base, "POST", "/api/v1/users", {
"username": "b" + uuid.uuid4().hex[:20],
"display_name": "Benchmark",
})
users.append(user)
chat = request(opener, base, "POST", "/api/v1/chats", {
"title": "Benchmark",
"member_ids": [user["id"] for user in users],
}, users[0]["id"])
for index in range(history):
request(opener, base, "POST", f"/api/v1/chats/{chat['id']}/messages", {
"text": f"history-{index}",
}, users[index % len(users)]["id"])
return chat, users
def launch(mode, port, overlay):
command = ["go", "run"]
if mode == "baseline":
command.extend(["-overlay", str(overlay)])
command.append("./cmd/messenger")
process = subprocess.Popen(
command,
cwd=ROOT,
env={**os.environ, "APP_HOST": "127.0.0.1", "APP_PORT": str(port)},
stdout=subprocess.DEVNULL,
stderr=subprocess.PIPE,
start_new_session=True,
)
return process, command
def run_case(mode, history, concurrency, requests, overlay):
port = available_port()
base = f"http://127.0.0.1:{port}"
opener = urllib.request.build_opener(urllib.request.ProxyHandler({}))
process, command = launch(mode, port, overlay)
try:
wait_ready(process, opener, base)
chat, users = setup(opener, base, history)
rss = []
stop_sampling = threading.Event()
sampling_errors = []
def sample():
while not stop_sampling.is_set():
try:
rss.append(server_rss_kib(process.pid))
except RuntimeError as exc:
sampling_errors.append(str(exc))
stop_sampling.wait(0.1)
sampler = threading.Thread(target=sample)
sampler.start()
warmup = min(100, max(10, requests // 100))
for index in range(warmup):
request(opener, base, "POST", f"/api/v1/chats/{chat['id']}/messages", {
"text": f"warmup-{index}",
}, users[index % len(users)]["id"])
started = time.monotonic()
def worker(worker_id):
latencies = []
failures = 0
worker_requests = requests // concurrency + (1 if worker_id < requests % concurrency else 0)
for index in range(worker_requests):
request_started = time.perf_counter()
try:
request(opener, base, "POST", f"/api/v1/chats/{chat['id']}/messages", {
"text": f"load-{worker_id}-{index}",
}, users[worker_id % len(users)]["id"])
latencies.append(time.perf_counter() - request_started)
except (OSError, ValueError, urllib.error.HTTPError):
failures += 1
index += 1
return latencies, failures
try:
with concurrent.futures.ThreadPoolExecutor(max_workers=concurrency) as pool:
results = list(pool.map(worker, range(concurrency)))
finally:
elapsed = time.monotonic() - started
stop_sampling.set()
sampler.join()
latencies = [value for values, _ in results for value in values]
failures = sum(value for _, value in results)
if not latencies or not rss or sampling_errors:
raise RuntimeError(f"invalid sample: successes={len(latencies)} rss={len(rss)} errors={sampling_errors}")
return {
"history": history,
"concurrency": concurrency,
"duration_seconds": round(elapsed, 3),
"requested_requests": requests,
"warmup_requests": warmup,
"successful_requests": len(latencies),
"failed_requests": failures,
"rps": round(len(latencies) / elapsed, 3),
"p50_ms": percentile(latencies, 0.50),
"p95_ms": percentile(latencies, 0.95),
"p99_ms": percentile(latencies, 0.99),
"rss_min_kib": min(rss),
"rss_max_kib": max(rss),
"rss_delta_kib": max(rss) - min(rss),
}, command
finally:
os.killpg(process.pid, signal.SIGTERM)
try:
process.wait(timeout=10)
except subprocess.TimeoutExpired:
os.killpg(process.pid, signal.SIGKILL)
process.wait()
def baseline_overlay(directory):
source = (ROOT / "internal/storage/memory/message_repository.go").resolve()
content = source.read_text()
if content.count(FAST_PATH) != 1:
raise RuntimeError("expected exactly one optimized append block")
replacement = directory / "message_repository_baseline.go"
replacement.write_text(content.replace(FAST_PATH, "", 1))
overlay = directory / "overlay.json"
overlay.write_text(json.dumps({"Replace": {str(source): str(replacement)}}))
return overlay
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--mode", choices=("baseline", "optimized"), required=True)
parser.add_argument("--requests", type=int, default=10000)
parser.add_argument("--output", type=Path, required=True)
args = parser.parse_args()
if args.requests <= 0:
parser.error("--requests must be positive")
with tempfile.TemporaryDirectory(prefix="messenger-track1-") as temp_dir:
overlay = baseline_overlay(Path(temp_dir))
rows = []
for history in (1000, 10000, 50000):
for concurrency in (1, 4, 16, 32):
row, command = run_case(args.mode, history, concurrency, args.requests, overlay)
rows.append(row)
print(json.dumps(row, ensure_ascii=False), flush=True)
result = {
"mode": args.mode,
"command": command,
"go_version": subprocess.check_output(["go", "version"], text=True).strip(),
"os_arch": subprocess.check_output(["go", "env", "GOOS", "GOARCH"], text=True).split(),
"requests_per_case": args.requests,
"rss_source": "ps RSS of go run child executable, KiB; sampled every 100 ms",
"rows": rows,
}
args.output.parent.mkdir(parents=True, exist_ok=True)
args.output.write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n")
if __name__ == "__main__":
main()