This commit is contained in:
Vladislav Kunetskiy
2026-10-01 12:59:06 +03:00
parent 65003b6119
commit 51b236a69f
86 changed files with 33332 additions and 16 deletions
+271
View File
@@ -0,0 +1,271 @@
import argparse
import json
import os
import subprocess
import threading
import time
import urllib.error
import urllib.request
import uuid
import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt
def request(base_url, method, path, body=None, user_id=None):
payload = json.dumps(body, ensure_ascii=False).encode() if body is not None else None
headers = {"Content-Type": "application/json"} if payload else {}
if user_id:
headers["X-User-Id"] = user_id
request = urllib.request.Request(
base_url + path,
data=payload,
headers=headers,
method=method,
)
with urllib.request.urlopen(request, timeout=3) as response:
return json.loads(response.read())
def wait_ready(base_url):
deadline = time.monotonic() + 20
while time.monotonic() < deadline:
try:
request(base_url, "GET", "/health/live")
return
except OSError:
time.sleep(0.1)
raise RuntimeError("container did not become ready")
def process_alive(container):
result = subprocess.run(
["docker", "inspect", "--format", "{{.State.Running}}", container],
capture_output=True,
text=True,
check=False,
)
return result.stdout.strip() == "true"
def sample_stats(container, samples, stop):
while not stop.is_set():
result = subprocess.run(
["docker", "stats", "--no-stream", "--format", "{{json .}}", container],
capture_output=True,
text=True,
check=False,
)
if result.returncode == 0 and result.stdout.strip():
try:
value = json.loads(result.stdout)
memory = value.get("MemUsage", "").split("/")[0].strip()
if memory == "0B" or not process_alive(container):
continue
samples.append({
"time_seconds": time.monotonic(),
"memory": memory,
"memory_mib": parse_memory(memory),
})
except (ValueError, IndexError):
pass
stop.wait(0.1)
def parse_memory(value):
units = {"B": 1 / 1024 / 1024, "KiB": 1 / 1024, "MiB": 1, "GiB": 1024}
for unit, factor in sorted(units.items(), key=lambda item: -len(item[0])):
if value.endswith(unit):
return float(value[:-len(unit)].strip()) * factor
raise ValueError(value)
def make_plot(data, output):
samples = [sample for sample in data["memory_samples"] if sample["memory_mib"] > 0]
start = samples[0]["time_seconds"] if samples else 0
x = [sample["time_seconds"] - start for sample in samples]
memory = [sample["memory_mib"] for sample in samples]
fig, memory_axis = plt.subplots(figsize=(11, 6))
memory_axis.plot(x, memory, color="tab:red", label="RSS / container memory")
memory_axis.axhline(data["memory_limit_mib"], color="black", linestyle="--", label="memory limit")
memory_axis.set_xlabel("Time, seconds")
memory_axis.set_ylabel("Memory, MiB")
memory_axis.grid(True, alpha=0.25)
message_axis = memory_axis.twinx()
message_axis.plot(
data["message_points_x"],
data["message_points_y"],
color="tab:blue",
label="successful messages",
)
message_axis.set_ylabel("Successful messages")
lines, labels = memory_axis.get_legend_handles_labels()
other_lines, other_labels = message_axis.get_legend_handles_labels()
memory_axis.legend(lines + other_lines, labels + other_labels, loc="upper left")
memory_axis.set_title("In-memory messenger: memory exhaustion point")
if data["oom_killed"] and data["failure_time_seconds"] is not None:
memory_axis.axvline(
data["failure_time_seconds"],
color="tab:purple",
linestyle=":",
label="container OOMKilled",
)
memory_axis.annotate(
"OOMKilled",
(data["failure_time_seconds"], data["memory_limit_mib"]),
xytext=(8, -18),
textcoords="offset points",
color="tab:purple",
)
fig.tight_layout()
fig.savefig(output, dpi=160)
plt.close(fig)
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--image", required=True)
parser.add_argument("--memory", default="32m")
parser.add_argument("--output", default="evidence/memory-experiment.json")
parser.add_argument("--plot", default="evidence/memory-experiment.png")
parser.add_argument("--max-messages", type=int, default=100000)
args = parser.parse_args()
container = "mtusi-memory-experiment-" + uuid.uuid4().hex[:8]
port = 18081
base_url = f"http://127.0.0.1:{port}"
subprocess.run(
[
"docker", "run", "-d", "--name", container,
"--memory", args.memory, "--memory-swap", args.memory,
"-p", f"127.0.0.1:{port}:8080", args.image,
],
check=True,
capture_output=True,
text=True,
)
samples = []
stop_sampling = threading.Event()
sampler = threading.Thread(target=sample_stats, args=(container, samples, stop_sampling))
successful = 0
failures = 0
fatal_error = None
started = time.monotonic()
message_points_x = []
message_points_y = []
failure_time = None
oom_killed = False
last_error = None
last_state = {}
logs = ""
try:
wait_ready(base_url)
user = request(base_url, "POST", "/api/v1/users", {
"username": "memory_" + uuid.uuid4().hex[:16],
"display_name": "Memory experiment",
})
chat = request(base_url, "POST", "/api/v1/chats", {
"title": "Memory experiment",
"member_ids": [user["id"]],
}, user["id"])
sampler.start()
time.sleep(0.5)
body = "x" * 4000
for index in range(args.max_messages):
try:
request(
base_url,
"POST",
f"/api/v1/chats/{chat['id']}/messages",
{"text": body},
user["id"],
)
successful += 1
message_points_x.append(time.monotonic() - started)
message_points_y.append(successful)
except (OSError, urllib.error.HTTPError, ValueError) as exc:
last_error = type(exc).__name__ + ": " + str(exc)
failures += 1
failure_time = time.monotonic() - started
break
time.sleep(0.5)
if not process_alive(container):
failure_time = failure_time or (time.monotonic() - started)
inspect = subprocess.run(
["docker", "inspect", "--format", "{{json .State}}", container],
capture_output=True,
text=True,
check=False,
)
state = json.loads(inspect.stdout) if inspect.stdout.strip() else {}
if not state:
state_result = subprocess.run(
["docker", "inspect", "--format", "{{json .State}}", container],
capture_output=True,
text=True,
check=False,
)
state = json.loads(state_result.stdout) if state_result.stdout.strip() else {}
last_state = state
logs_result = subprocess.run(["docker", "logs", container], capture_output=True, text=True, check=False)
logs = logs_result.stdout + logs_result.stderr
oom_killed = bool(state.get("OOMKilled"))
if failure_time is None and failures:
failure_time = time.monotonic() - started
except Exception as exc:
fatal_error = type(exc).__name__ + ": " + str(exc)
failure_time = failure_time or (time.monotonic() - started)
finally:
stop_sampling.set()
if sampler.is_alive():
sampler.join(timeout=2)
memory_limit_mib = float(args.memory.rstrip("mM")) if args.memory.lower().endswith("m") else 0
if not oom_killed:
inspect = subprocess.run(
["docker", "inspect", "--format", "{{json .State}}", container],
capture_output=True,
text=True,
check=False,
)
state = json.loads(inspect.stdout) if inspect.stdout.strip() else {}
last_state = state
oom_killed = bool(state.get("OOMKilled"))
result = {
"image": args.image,
"memory_limit": args.memory,
"memory_limit_mib": memory_limit_mib,
"successful_messages": successful,
"failed_requests": failures,
"oom_killed": oom_killed,
"last_error": last_error,
"fatal_error": fatal_error,
"container_state": last_state,
"container_logs_tail": logs[-20000:],
"failure_time_seconds": failure_time,
"memory_samples": samples,
"message_points_x": message_points_x,
"message_points_y": message_points_y,
"reason": "process stopped after the container memory limit was reached" if oom_killed else "request processing did not reach the memory limit",
}
with open(args.output, "w", encoding="utf-8") as output:
json.dump(result, output, ensure_ascii=False, indent=2)
make_plot(result, args.plot)
print(json.dumps({key: result[key] for key in (
"memory_limit", "successful_messages", "failed_requests", "oom_killed", "failure_time_seconds", "reason",
)}, ensure_ascii=False, indent=2))
subprocess.run(["docker", "rm", "-f", container], capture_output=True, check=False)
if __name__ == "__main__":
main()