272 lines
9.5 KiB
Python
272 lines
9.5 KiB
Python
import argparse
|
|
import json
|
|
import os
|
|
import subprocess
|
|
import threading
|
|
import time
|
|
import urllib.error
|
|
import urllib.request
|
|
import uuid
|
|
|
|
import matplotlib
|
|
|
|
matplotlib.use("Agg")
|
|
import matplotlib.pyplot as plt
|
|
|
|
|
|
def request(base_url, method, path, body=None, user_id=None):
|
|
payload = json.dumps(body, ensure_ascii=False).encode() if body is not None else None
|
|
headers = {"Content-Type": "application/json"} if payload else {}
|
|
if user_id:
|
|
headers["X-User-Id"] = user_id
|
|
request = urllib.request.Request(
|
|
base_url + path,
|
|
data=payload,
|
|
headers=headers,
|
|
method=method,
|
|
)
|
|
with urllib.request.urlopen(request, timeout=3) as response:
|
|
return json.loads(response.read())
|
|
|
|
|
|
def wait_ready(base_url):
|
|
deadline = time.monotonic() + 20
|
|
while time.monotonic() < deadline:
|
|
try:
|
|
request(base_url, "GET", "/health/live")
|
|
return
|
|
except OSError:
|
|
time.sleep(0.1)
|
|
raise RuntimeError("container did not become ready")
|
|
|
|
|
|
def process_alive(container):
|
|
result = subprocess.run(
|
|
["docker", "inspect", "--format", "{{.State.Running}}", container],
|
|
capture_output=True,
|
|
text=True,
|
|
check=False,
|
|
)
|
|
return result.stdout.strip() == "true"
|
|
|
|
|
|
def sample_stats(container, samples, stop):
|
|
while not stop.is_set():
|
|
result = subprocess.run(
|
|
["docker", "stats", "--no-stream", "--format", "{{json .}}", container],
|
|
capture_output=True,
|
|
text=True,
|
|
check=False,
|
|
)
|
|
if result.returncode == 0 and result.stdout.strip():
|
|
try:
|
|
value = json.loads(result.stdout)
|
|
memory = value.get("MemUsage", "").split("/")[0].strip()
|
|
if memory == "0B" or not process_alive(container):
|
|
continue
|
|
samples.append({
|
|
"time_seconds": time.monotonic(),
|
|
"memory": memory,
|
|
"memory_mib": parse_memory(memory),
|
|
})
|
|
except (ValueError, IndexError):
|
|
pass
|
|
stop.wait(0.1)
|
|
|
|
|
|
def parse_memory(value):
|
|
units = {"B": 1 / 1024 / 1024, "KiB": 1 / 1024, "MiB": 1, "GiB": 1024}
|
|
for unit, factor in sorted(units.items(), key=lambda item: -len(item[0])):
|
|
if value.endswith(unit):
|
|
return float(value[:-len(unit)].strip()) * factor
|
|
raise ValueError(value)
|
|
|
|
|
|
def make_plot(data, output):
|
|
samples = [sample for sample in data["memory_samples"] if sample["memory_mib"] > 0]
|
|
start = samples[0]["time_seconds"] if samples else 0
|
|
x = [sample["time_seconds"] - start for sample in samples]
|
|
memory = [sample["memory_mib"] for sample in samples]
|
|
|
|
fig, memory_axis = plt.subplots(figsize=(11, 6))
|
|
memory_axis.plot(x, memory, color="tab:red", label="RSS / container memory")
|
|
memory_axis.axhline(data["memory_limit_mib"], color="black", linestyle="--", label="memory limit")
|
|
memory_axis.set_xlabel("Time, seconds")
|
|
memory_axis.set_ylabel("Memory, MiB")
|
|
memory_axis.grid(True, alpha=0.25)
|
|
|
|
message_axis = memory_axis.twinx()
|
|
message_axis.plot(
|
|
data["message_points_x"],
|
|
data["message_points_y"],
|
|
color="tab:blue",
|
|
label="successful messages",
|
|
)
|
|
message_axis.set_ylabel("Successful messages")
|
|
|
|
lines, labels = memory_axis.get_legend_handles_labels()
|
|
other_lines, other_labels = message_axis.get_legend_handles_labels()
|
|
memory_axis.legend(lines + other_lines, labels + other_labels, loc="upper left")
|
|
memory_axis.set_title("In-memory messenger: memory exhaustion point")
|
|
if data["oom_killed"] and data["failure_time_seconds"] is not None:
|
|
memory_axis.axvline(
|
|
data["failure_time_seconds"],
|
|
color="tab:purple",
|
|
linestyle=":",
|
|
label="container OOMKilled",
|
|
)
|
|
memory_axis.annotate(
|
|
"OOMKilled",
|
|
(data["failure_time_seconds"], data["memory_limit_mib"]),
|
|
xytext=(8, -18),
|
|
textcoords="offset points",
|
|
color="tab:purple",
|
|
)
|
|
fig.tight_layout()
|
|
fig.savefig(output, dpi=160)
|
|
plt.close(fig)
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("--image", required=True)
|
|
parser.add_argument("--memory", default="32m")
|
|
parser.add_argument("--output", default="evidence/memory-experiment.json")
|
|
parser.add_argument("--plot", default="evidence/memory-experiment.png")
|
|
parser.add_argument("--max-messages", type=int, default=100000)
|
|
args = parser.parse_args()
|
|
|
|
container = "mtusi-memory-experiment-" + uuid.uuid4().hex[:8]
|
|
port = 18081
|
|
base_url = f"http://127.0.0.1:{port}"
|
|
subprocess.run(
|
|
[
|
|
"docker", "run", "-d", "--name", container,
|
|
"--memory", args.memory, "--memory-swap", args.memory,
|
|
"-p", f"127.0.0.1:{port}:8080", args.image,
|
|
],
|
|
check=True,
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
|
|
samples = []
|
|
stop_sampling = threading.Event()
|
|
sampler = threading.Thread(target=sample_stats, args=(container, samples, stop_sampling))
|
|
successful = 0
|
|
failures = 0
|
|
fatal_error = None
|
|
started = time.monotonic()
|
|
message_points_x = []
|
|
message_points_y = []
|
|
failure_time = None
|
|
oom_killed = False
|
|
last_error = None
|
|
last_state = {}
|
|
logs = ""
|
|
|
|
try:
|
|
wait_ready(base_url)
|
|
user = request(base_url, "POST", "/api/v1/users", {
|
|
"username": "memory_" + uuid.uuid4().hex[:16],
|
|
"display_name": "Memory experiment",
|
|
})
|
|
chat = request(base_url, "POST", "/api/v1/chats", {
|
|
"title": "Memory experiment",
|
|
"member_ids": [user["id"]],
|
|
}, user["id"])
|
|
sampler.start()
|
|
time.sleep(0.5)
|
|
|
|
body = "x" * 4000
|
|
for index in range(args.max_messages):
|
|
try:
|
|
request(
|
|
base_url,
|
|
"POST",
|
|
f"/api/v1/chats/{chat['id']}/messages",
|
|
{"text": body},
|
|
user["id"],
|
|
)
|
|
successful += 1
|
|
message_points_x.append(time.monotonic() - started)
|
|
message_points_y.append(successful)
|
|
except (OSError, urllib.error.HTTPError, ValueError) as exc:
|
|
last_error = type(exc).__name__ + ": " + str(exc)
|
|
failures += 1
|
|
failure_time = time.monotonic() - started
|
|
break
|
|
|
|
time.sleep(0.5)
|
|
if not process_alive(container):
|
|
failure_time = failure_time or (time.monotonic() - started)
|
|
inspect = subprocess.run(
|
|
["docker", "inspect", "--format", "{{json .State}}", container],
|
|
capture_output=True,
|
|
text=True,
|
|
check=False,
|
|
)
|
|
state = json.loads(inspect.stdout) if inspect.stdout.strip() else {}
|
|
if not state:
|
|
state_result = subprocess.run(
|
|
["docker", "inspect", "--format", "{{json .State}}", container],
|
|
capture_output=True,
|
|
text=True,
|
|
check=False,
|
|
)
|
|
state = json.loads(state_result.stdout) if state_result.stdout.strip() else {}
|
|
last_state = state
|
|
logs_result = subprocess.run(["docker", "logs", container], capture_output=True, text=True, check=False)
|
|
logs = logs_result.stdout + logs_result.stderr
|
|
oom_killed = bool(state.get("OOMKilled"))
|
|
if failure_time is None and failures:
|
|
failure_time = time.monotonic() - started
|
|
except Exception as exc:
|
|
fatal_error = type(exc).__name__ + ": " + str(exc)
|
|
failure_time = failure_time or (time.monotonic() - started)
|
|
finally:
|
|
stop_sampling.set()
|
|
if sampler.is_alive():
|
|
sampler.join(timeout=2)
|
|
|
|
memory_limit_mib = float(args.memory.rstrip("mM")) if args.memory.lower().endswith("m") else 0
|
|
if not oom_killed:
|
|
inspect = subprocess.run(
|
|
["docker", "inspect", "--format", "{{json .State}}", container],
|
|
capture_output=True,
|
|
text=True,
|
|
check=False,
|
|
)
|
|
state = json.loads(inspect.stdout) if inspect.stdout.strip() else {}
|
|
last_state = state
|
|
oom_killed = bool(state.get("OOMKilled"))
|
|
|
|
result = {
|
|
"image": args.image,
|
|
"memory_limit": args.memory,
|
|
"memory_limit_mib": memory_limit_mib,
|
|
"successful_messages": successful,
|
|
"failed_requests": failures,
|
|
"oom_killed": oom_killed,
|
|
"last_error": last_error,
|
|
"fatal_error": fatal_error,
|
|
"container_state": last_state,
|
|
"container_logs_tail": logs[-20000:],
|
|
"failure_time_seconds": failure_time,
|
|
"memory_samples": samples,
|
|
"message_points_x": message_points_x,
|
|
"message_points_y": message_points_y,
|
|
"reason": "process stopped after the container memory limit was reached" if oom_killed else "request processing did not reach the memory limit",
|
|
}
|
|
with open(args.output, "w", encoding="utf-8") as output:
|
|
json.dump(result, output, ensure_ascii=False, indent=2)
|
|
make_plot(result, args.plot)
|
|
print(json.dumps({key: result[key] for key in (
|
|
"memory_limit", "successful_messages", "failed_requests", "oom_killed", "failure_time_seconds", "reason",
|
|
)}, ensure_ascii=False, indent=2))
|
|
subprocess.run(["docker", "rm", "-f", container], capture_output=True, check=False)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|