lab1
This commit is contained in:
@@ -0,0 +1,271 @@
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import threading
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
import uuid
|
||||
|
||||
import matplotlib
|
||||
|
||||
matplotlib.use("Agg")
|
||||
import matplotlib.pyplot as plt
|
||||
|
||||
|
||||
def request(base_url, method, path, body=None, user_id=None):
|
||||
payload = json.dumps(body, ensure_ascii=False).encode() if body is not None else None
|
||||
headers = {"Content-Type": "application/json"} if payload else {}
|
||||
if user_id:
|
||||
headers["X-User-Id"] = user_id
|
||||
request = urllib.request.Request(
|
||||
base_url + path,
|
||||
data=payload,
|
||||
headers=headers,
|
||||
method=method,
|
||||
)
|
||||
with urllib.request.urlopen(request, timeout=3) as response:
|
||||
return json.loads(response.read())
|
||||
|
||||
|
||||
def wait_ready(base_url):
|
||||
deadline = time.monotonic() + 20
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
request(base_url, "GET", "/health/live")
|
||||
return
|
||||
except OSError:
|
||||
time.sleep(0.1)
|
||||
raise RuntimeError("container did not become ready")
|
||||
|
||||
|
||||
def process_alive(container):
|
||||
result = subprocess.run(
|
||||
["docker", "inspect", "--format", "{{.State.Running}}", container],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
return result.stdout.strip() == "true"
|
||||
|
||||
|
||||
def sample_stats(container, samples, stop):
|
||||
while not stop.is_set():
|
||||
result = subprocess.run(
|
||||
["docker", "stats", "--no-stream", "--format", "{{json .}}", container],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
if result.returncode == 0 and result.stdout.strip():
|
||||
try:
|
||||
value = json.loads(result.stdout)
|
||||
memory = value.get("MemUsage", "").split("/")[0].strip()
|
||||
if memory == "0B" or not process_alive(container):
|
||||
continue
|
||||
samples.append({
|
||||
"time_seconds": time.monotonic(),
|
||||
"memory": memory,
|
||||
"memory_mib": parse_memory(memory),
|
||||
})
|
||||
except (ValueError, IndexError):
|
||||
pass
|
||||
stop.wait(0.1)
|
||||
|
||||
|
||||
def parse_memory(value):
|
||||
units = {"B": 1 / 1024 / 1024, "KiB": 1 / 1024, "MiB": 1, "GiB": 1024}
|
||||
for unit, factor in sorted(units.items(), key=lambda item: -len(item[0])):
|
||||
if value.endswith(unit):
|
||||
return float(value[:-len(unit)].strip()) * factor
|
||||
raise ValueError(value)
|
||||
|
||||
|
||||
def make_plot(data, output):
|
||||
samples = [sample for sample in data["memory_samples"] if sample["memory_mib"] > 0]
|
||||
start = samples[0]["time_seconds"] if samples else 0
|
||||
x = [sample["time_seconds"] - start for sample in samples]
|
||||
memory = [sample["memory_mib"] for sample in samples]
|
||||
|
||||
fig, memory_axis = plt.subplots(figsize=(11, 6))
|
||||
memory_axis.plot(x, memory, color="tab:red", label="RSS / container memory")
|
||||
memory_axis.axhline(data["memory_limit_mib"], color="black", linestyle="--", label="memory limit")
|
||||
memory_axis.set_xlabel("Time, seconds")
|
||||
memory_axis.set_ylabel("Memory, MiB")
|
||||
memory_axis.grid(True, alpha=0.25)
|
||||
|
||||
message_axis = memory_axis.twinx()
|
||||
message_axis.plot(
|
||||
data["message_points_x"],
|
||||
data["message_points_y"],
|
||||
color="tab:blue",
|
||||
label="successful messages",
|
||||
)
|
||||
message_axis.set_ylabel("Successful messages")
|
||||
|
||||
lines, labels = memory_axis.get_legend_handles_labels()
|
||||
other_lines, other_labels = message_axis.get_legend_handles_labels()
|
||||
memory_axis.legend(lines + other_lines, labels + other_labels, loc="upper left")
|
||||
memory_axis.set_title("In-memory messenger: memory exhaustion point")
|
||||
if data["oom_killed"] and data["failure_time_seconds"] is not None:
|
||||
memory_axis.axvline(
|
||||
data["failure_time_seconds"],
|
||||
color="tab:purple",
|
||||
linestyle=":",
|
||||
label="container OOMKilled",
|
||||
)
|
||||
memory_axis.annotate(
|
||||
"OOMKilled",
|
||||
(data["failure_time_seconds"], data["memory_limit_mib"]),
|
||||
xytext=(8, -18),
|
||||
textcoords="offset points",
|
||||
color="tab:purple",
|
||||
)
|
||||
fig.tight_layout()
|
||||
fig.savefig(output, dpi=160)
|
||||
plt.close(fig)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--image", required=True)
|
||||
parser.add_argument("--memory", default="32m")
|
||||
parser.add_argument("--output", default="evidence/memory-experiment.json")
|
||||
parser.add_argument("--plot", default="evidence/memory-experiment.png")
|
||||
parser.add_argument("--max-messages", type=int, default=100000)
|
||||
args = parser.parse_args()
|
||||
|
||||
container = "mtusi-memory-experiment-" + uuid.uuid4().hex[:8]
|
||||
port = 18081
|
||||
base_url = f"http://127.0.0.1:{port}"
|
||||
subprocess.run(
|
||||
[
|
||||
"docker", "run", "-d", "--name", container,
|
||||
"--memory", args.memory, "--memory-swap", args.memory,
|
||||
"-p", f"127.0.0.1:{port}:8080", args.image,
|
||||
],
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
|
||||
samples = []
|
||||
stop_sampling = threading.Event()
|
||||
sampler = threading.Thread(target=sample_stats, args=(container, samples, stop_sampling))
|
||||
successful = 0
|
||||
failures = 0
|
||||
fatal_error = None
|
||||
started = time.monotonic()
|
||||
message_points_x = []
|
||||
message_points_y = []
|
||||
failure_time = None
|
||||
oom_killed = False
|
||||
last_error = None
|
||||
last_state = {}
|
||||
logs = ""
|
||||
|
||||
try:
|
||||
wait_ready(base_url)
|
||||
user = request(base_url, "POST", "/api/v1/users", {
|
||||
"username": "memory_" + uuid.uuid4().hex[:16],
|
||||
"display_name": "Memory experiment",
|
||||
})
|
||||
chat = request(base_url, "POST", "/api/v1/chats", {
|
||||
"title": "Memory experiment",
|
||||
"member_ids": [user["id"]],
|
||||
}, user["id"])
|
||||
sampler.start()
|
||||
time.sleep(0.5)
|
||||
|
||||
body = "x" * 4000
|
||||
for index in range(args.max_messages):
|
||||
try:
|
||||
request(
|
||||
base_url,
|
||||
"POST",
|
||||
f"/api/v1/chats/{chat['id']}/messages",
|
||||
{"text": body},
|
||||
user["id"],
|
||||
)
|
||||
successful += 1
|
||||
message_points_x.append(time.monotonic() - started)
|
||||
message_points_y.append(successful)
|
||||
except (OSError, urllib.error.HTTPError, ValueError) as exc:
|
||||
last_error = type(exc).__name__ + ": " + str(exc)
|
||||
failures += 1
|
||||
failure_time = time.monotonic() - started
|
||||
break
|
||||
|
||||
time.sleep(0.5)
|
||||
if not process_alive(container):
|
||||
failure_time = failure_time or (time.monotonic() - started)
|
||||
inspect = subprocess.run(
|
||||
["docker", "inspect", "--format", "{{json .State}}", container],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
state = json.loads(inspect.stdout) if inspect.stdout.strip() else {}
|
||||
if not state:
|
||||
state_result = subprocess.run(
|
||||
["docker", "inspect", "--format", "{{json .State}}", container],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
state = json.loads(state_result.stdout) if state_result.stdout.strip() else {}
|
||||
last_state = state
|
||||
logs_result = subprocess.run(["docker", "logs", container], capture_output=True, text=True, check=False)
|
||||
logs = logs_result.stdout + logs_result.stderr
|
||||
oom_killed = bool(state.get("OOMKilled"))
|
||||
if failure_time is None and failures:
|
||||
failure_time = time.monotonic() - started
|
||||
except Exception as exc:
|
||||
fatal_error = type(exc).__name__ + ": " + str(exc)
|
||||
failure_time = failure_time or (time.monotonic() - started)
|
||||
finally:
|
||||
stop_sampling.set()
|
||||
if sampler.is_alive():
|
||||
sampler.join(timeout=2)
|
||||
|
||||
memory_limit_mib = float(args.memory.rstrip("mM")) if args.memory.lower().endswith("m") else 0
|
||||
if not oom_killed:
|
||||
inspect = subprocess.run(
|
||||
["docker", "inspect", "--format", "{{json .State}}", container],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
state = json.loads(inspect.stdout) if inspect.stdout.strip() else {}
|
||||
last_state = state
|
||||
oom_killed = bool(state.get("OOMKilled"))
|
||||
|
||||
result = {
|
||||
"image": args.image,
|
||||
"memory_limit": args.memory,
|
||||
"memory_limit_mib": memory_limit_mib,
|
||||
"successful_messages": successful,
|
||||
"failed_requests": failures,
|
||||
"oom_killed": oom_killed,
|
||||
"last_error": last_error,
|
||||
"fatal_error": fatal_error,
|
||||
"container_state": last_state,
|
||||
"container_logs_tail": logs[-20000:],
|
||||
"failure_time_seconds": failure_time,
|
||||
"memory_samples": samples,
|
||||
"message_points_x": message_points_x,
|
||||
"message_points_y": message_points_y,
|
||||
"reason": "process stopped after the container memory limit was reached" if oom_killed else "request processing did not reach the memory limit",
|
||||
}
|
||||
with open(args.output, "w", encoding="utf-8") as output:
|
||||
json.dump(result, output, ensure_ascii=False, indent=2)
|
||||
make_plot(result, args.plot)
|
||||
print(json.dumps({key: result[key] for key in (
|
||||
"memory_limit", "successful_messages", "failed_requests", "oom_killed", "failure_time_seconds", "reason",
|
||||
)}, ensure_ascii=False, indent=2))
|
||||
subprocess.run(["docker", "rm", "-f", container], capture_output=True, check=False)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,267 @@
|
||||
import argparse
|
||||
import concurrent.futures
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import signal
|
||||
import socket
|
||||
import subprocess
|
||||
import tempfile
|
||||
import threading
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
import uuid
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
FAST_PATH = """\tif len(history) == 0 || compareKeyset(
|
||||
\t\thistory[len(history)-1].CreatedAt,
|
||||
\t\thistory[len(history)-1].ID,
|
||||
\t\tmessage.CreatedAt,
|
||||
\t\tmessage.ID,
|
||||
\t) < 0 {
|
||||
\t\ts.messagesByChat[message.ChatID] = append(history, message)
|
||||
|
||||
\t\treturn message, nil
|
||||
\t}
|
||||
|
||||
"""
|
||||
|
||||
|
||||
def request(opener, base, method, path, body=None, user=None):
|
||||
data = json.dumps(body, ensure_ascii=False).encode() if body is not None else None
|
||||
headers = {}
|
||||
if data is not None:
|
||||
headers["Content-Type"] = "application/json"
|
||||
if user is not None:
|
||||
headers["X-User-Id"] = user
|
||||
req = urllib.request.Request(base + path, data=data, headers=headers, method=method)
|
||||
with opener.open(req, timeout=10) as response:
|
||||
return json.loads(response.read())
|
||||
|
||||
|
||||
def available_port():
|
||||
with socket.socket() as listener:
|
||||
listener.bind(("127.0.0.1", 0))
|
||||
return listener.getsockname()[1]
|
||||
|
||||
|
||||
def descendants(root_pid):
|
||||
result = subprocess.run(
|
||||
["ps", "-axo", "pid=,ppid=,rss=,command="],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=True,
|
||||
)
|
||||
entries = []
|
||||
for line in result.stdout.splitlines():
|
||||
fields = line.strip().split(None, 3)
|
||||
if len(fields) == 4:
|
||||
try:
|
||||
entries.append((int(fields[0]), int(fields[1]), int(fields[2]), fields[3]))
|
||||
except ValueError:
|
||||
continue
|
||||
children = {root_pid}
|
||||
while True:
|
||||
discovered = children | {pid for pid, ppid, _, _ in entries if ppid in children}
|
||||
if discovered == children:
|
||||
break
|
||||
children = discovered
|
||||
return [entry for entry in entries if entry[0] in children and entry[0] != root_pid]
|
||||
|
||||
|
||||
def server_rss_kib(go_run_pid):
|
||||
servers = [rss for _, _, rss, command in descendants(go_run_pid) if command.rstrip().endswith("/messenger")]
|
||||
if not servers:
|
||||
raise RuntimeError("go run server process was not found")
|
||||
return max(servers)
|
||||
|
||||
|
||||
def wait_ready(process, opener, base):
|
||||
deadline = time.monotonic() + 60
|
||||
while time.monotonic() < deadline:
|
||||
if process.poll() is not None:
|
||||
raise RuntimeError(f"go run exited with code {process.returncode}")
|
||||
try:
|
||||
request(opener, base, "GET", "/health/live")
|
||||
server_rss_kib(process.pid)
|
||||
return
|
||||
except (OSError, RuntimeError):
|
||||
time.sleep(0.1)
|
||||
raise RuntimeError("go run server did not become ready")
|
||||
|
||||
|
||||
def percentile(latencies, fraction):
|
||||
ordered = sorted(latencies)
|
||||
index = max(0, min(len(ordered) - 1, (len(ordered) * int(fraction * 100) + 99) // 100 - 1))
|
||||
return round(ordered[index] * 1000, 3)
|
||||
|
||||
|
||||
def setup(opener, base, history):
|
||||
users = []
|
||||
for _ in range(10):
|
||||
user = request(opener, base, "POST", "/api/v1/users", {
|
||||
"username": "b" + uuid.uuid4().hex[:20],
|
||||
"display_name": "Benchmark",
|
||||
})
|
||||
users.append(user)
|
||||
|
||||
chat = request(opener, base, "POST", "/api/v1/chats", {
|
||||
"title": "Benchmark",
|
||||
"member_ids": [user["id"] for user in users],
|
||||
}, users[0]["id"])
|
||||
|
||||
for index in range(history):
|
||||
request(opener, base, "POST", f"/api/v1/chats/{chat['id']}/messages", {
|
||||
"text": f"history-{index}",
|
||||
}, users[index % len(users)]["id"])
|
||||
|
||||
return chat, users
|
||||
|
||||
|
||||
def launch(mode, port, overlay):
|
||||
command = ["go", "run"]
|
||||
if mode == "baseline":
|
||||
command.extend(["-overlay", str(overlay)])
|
||||
command.append("./cmd/messenger")
|
||||
process = subprocess.Popen(
|
||||
command,
|
||||
cwd=ROOT,
|
||||
env={**os.environ, "APP_HOST": "127.0.0.1", "APP_PORT": str(port)},
|
||||
stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.PIPE,
|
||||
start_new_session=True,
|
||||
)
|
||||
return process, command
|
||||
|
||||
|
||||
def run_case(mode, history, concurrency, requests, overlay):
|
||||
port = available_port()
|
||||
base = f"http://127.0.0.1:{port}"
|
||||
opener = urllib.request.build_opener(urllib.request.ProxyHandler({}))
|
||||
process, command = launch(mode, port, overlay)
|
||||
try:
|
||||
wait_ready(process, opener, base)
|
||||
chat, users = setup(opener, base, history)
|
||||
rss = []
|
||||
stop_sampling = threading.Event()
|
||||
sampling_errors = []
|
||||
|
||||
def sample():
|
||||
while not stop_sampling.is_set():
|
||||
try:
|
||||
rss.append(server_rss_kib(process.pid))
|
||||
except RuntimeError as exc:
|
||||
sampling_errors.append(str(exc))
|
||||
stop_sampling.wait(0.1)
|
||||
|
||||
sampler = threading.Thread(target=sample)
|
||||
sampler.start()
|
||||
warmup = min(100, max(10, requests // 100))
|
||||
|
||||
for index in range(warmup):
|
||||
request(opener, base, "POST", f"/api/v1/chats/{chat['id']}/messages", {
|
||||
"text": f"warmup-{index}",
|
||||
}, users[index % len(users)]["id"])
|
||||
|
||||
started = time.monotonic()
|
||||
|
||||
def worker(worker_id):
|
||||
latencies = []
|
||||
failures = 0
|
||||
worker_requests = requests // concurrency + (1 if worker_id < requests % concurrency else 0)
|
||||
for index in range(worker_requests):
|
||||
request_started = time.perf_counter()
|
||||
try:
|
||||
request(opener, base, "POST", f"/api/v1/chats/{chat['id']}/messages", {
|
||||
"text": f"load-{worker_id}-{index}",
|
||||
}, users[worker_id % len(users)]["id"])
|
||||
latencies.append(time.perf_counter() - request_started)
|
||||
except (OSError, ValueError, urllib.error.HTTPError):
|
||||
failures += 1
|
||||
index += 1
|
||||
return latencies, failures
|
||||
|
||||
try:
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=concurrency) as pool:
|
||||
results = list(pool.map(worker, range(concurrency)))
|
||||
finally:
|
||||
elapsed = time.monotonic() - started
|
||||
stop_sampling.set()
|
||||
sampler.join()
|
||||
|
||||
latencies = [value for values, _ in results for value in values]
|
||||
failures = sum(value for _, value in results)
|
||||
if not latencies or not rss or sampling_errors:
|
||||
raise RuntimeError(f"invalid sample: successes={len(latencies)} rss={len(rss)} errors={sampling_errors}")
|
||||
|
||||
return {
|
||||
"history": history,
|
||||
"concurrency": concurrency,
|
||||
"duration_seconds": round(elapsed, 3),
|
||||
"requested_requests": requests,
|
||||
"warmup_requests": warmup,
|
||||
"successful_requests": len(latencies),
|
||||
"failed_requests": failures,
|
||||
"rps": round(len(latencies) / elapsed, 3),
|
||||
"p50_ms": percentile(latencies, 0.50),
|
||||
"p95_ms": percentile(latencies, 0.95),
|
||||
"p99_ms": percentile(latencies, 0.99),
|
||||
"rss_min_kib": min(rss),
|
||||
"rss_max_kib": max(rss),
|
||||
"rss_delta_kib": max(rss) - min(rss),
|
||||
}, command
|
||||
finally:
|
||||
os.killpg(process.pid, signal.SIGTERM)
|
||||
try:
|
||||
process.wait(timeout=10)
|
||||
except subprocess.TimeoutExpired:
|
||||
os.killpg(process.pid, signal.SIGKILL)
|
||||
process.wait()
|
||||
|
||||
|
||||
def baseline_overlay(directory):
|
||||
source = (ROOT / "internal/storage/memory/message_repository.go").resolve()
|
||||
content = source.read_text()
|
||||
if content.count(FAST_PATH) != 1:
|
||||
raise RuntimeError("expected exactly one optimized append block")
|
||||
replacement = directory / "message_repository_baseline.go"
|
||||
replacement.write_text(content.replace(FAST_PATH, "", 1))
|
||||
overlay = directory / "overlay.json"
|
||||
overlay.write_text(json.dumps({"Replace": {str(source): str(replacement)}}))
|
||||
return overlay
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--mode", choices=("baseline", "optimized"), required=True)
|
||||
parser.add_argument("--requests", type=int, default=10000)
|
||||
parser.add_argument("--output", type=Path, required=True)
|
||||
args = parser.parse_args()
|
||||
if args.requests <= 0:
|
||||
parser.error("--requests must be positive")
|
||||
|
||||
with tempfile.TemporaryDirectory(prefix="messenger-track1-") as temp_dir:
|
||||
overlay = baseline_overlay(Path(temp_dir))
|
||||
rows = []
|
||||
for history in (1000, 10000, 50000):
|
||||
for concurrency in (1, 4, 16, 32):
|
||||
row, command = run_case(args.mode, history, concurrency, args.requests, overlay)
|
||||
rows.append(row)
|
||||
print(json.dumps(row, ensure_ascii=False), flush=True)
|
||||
result = {
|
||||
"mode": args.mode,
|
||||
"command": command,
|
||||
"go_version": subprocess.check_output(["go", "version"], text=True).strip(),
|
||||
"os_arch": subprocess.check_output(["go", "env", "GOOS", "GOARCH"], text=True).split(),
|
||||
"requests_per_case": args.requests,
|
||||
"rss_source": "ps RSS of go run child executable, KiB; sampled every 100 ms",
|
||||
"rows": rows,
|
||||
}
|
||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.output.write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user