import argparse import json import os import subprocess import threading import time import urllib.error import urllib.request import uuid import matplotlib matplotlib.use("Agg") import matplotlib.pyplot as plt def request(base_url, method, path, body=None, user_id=None): payload = json.dumps(body, ensure_ascii=False).encode() if body is not None else None headers = {"Content-Type": "application/json"} if payload else {} if user_id: headers["X-User-Id"] = user_id request = urllib.request.Request( base_url + path, data=payload, headers=headers, method=method, ) with urllib.request.urlopen(request, timeout=3) as response: return json.loads(response.read()) def wait_ready(base_url): deadline = time.monotonic() + 20 while time.monotonic() < deadline: try: request(base_url, "GET", "/health/live") return except OSError: time.sleep(0.1) raise RuntimeError("container did not become ready") def process_alive(container): result = subprocess.run( ["docker", "inspect", "--format", "{{.State.Running}}", container], capture_output=True, text=True, check=False, ) return result.stdout.strip() == "true" def sample_stats(container, samples, stop): while not stop.is_set(): result = subprocess.run( ["docker", "stats", "--no-stream", "--format", "{{json .}}", container], capture_output=True, text=True, check=False, ) if result.returncode == 0 and result.stdout.strip(): try: value = json.loads(result.stdout) memory = value.get("MemUsage", "").split("/")[0].strip() if memory == "0B" or not process_alive(container): continue samples.append({ "time_seconds": time.monotonic(), "memory": memory, "memory_mib": parse_memory(memory), }) except (ValueError, IndexError): pass stop.wait(0.1) def parse_memory(value): units = {"B": 1 / 1024 / 1024, "KiB": 1 / 1024, "MiB": 1, "GiB": 1024} for unit, factor in sorted(units.items(), key=lambda item: -len(item[0])): if value.endswith(unit): return float(value[:-len(unit)].strip()) * factor raise ValueError(value) def make_plot(data, output): samples = [sample for sample in data["memory_samples"] if sample["memory_mib"] > 0] start = samples[0]["time_seconds"] if samples else 0 x = [sample["time_seconds"] - start for sample in samples] memory = [sample["memory_mib"] for sample in samples] fig, memory_axis = plt.subplots(figsize=(11, 6)) memory_axis.plot(x, memory, color="tab:red", label="RSS / container memory") memory_axis.axhline(data["memory_limit_mib"], color="black", linestyle="--", label="memory limit") memory_axis.set_xlabel("Time, seconds") memory_axis.set_ylabel("Memory, MiB") memory_axis.grid(True, alpha=0.25) message_axis = memory_axis.twinx() message_axis.plot( data["message_points_x"], data["message_points_y"], color="tab:blue", label="successful messages", ) message_axis.set_ylabel("Successful messages") lines, labels = memory_axis.get_legend_handles_labels() other_lines, other_labels = message_axis.get_legend_handles_labels() memory_axis.legend(lines + other_lines, labels + other_labels, loc="upper left") memory_axis.set_title("In-memory messenger: memory exhaustion point") if data["oom_killed"] and data["failure_time_seconds"] is not None: memory_axis.axvline( data["failure_time_seconds"], color="tab:purple", linestyle=":", label="container OOMKilled", ) memory_axis.annotate( "OOMKilled", (data["failure_time_seconds"], data["memory_limit_mib"]), xytext=(8, -18), textcoords="offset points", color="tab:purple", ) fig.tight_layout() fig.savefig(output, dpi=160) plt.close(fig) def main(): parser = argparse.ArgumentParser() parser.add_argument("--image", required=True) parser.add_argument("--memory", default="32m") parser.add_argument("--output", default="evidence/memory-experiment.json") parser.add_argument("--plot", default="evidence/memory-experiment.png") parser.add_argument("--max-messages", type=int, default=100000) args = parser.parse_args() container = "mtusi-memory-experiment-" + uuid.uuid4().hex[:8] port = 18081 base_url = f"http://127.0.0.1:{port}" subprocess.run( [ "docker", "run", "-d", "--name", container, "--memory", args.memory, "--memory-swap", args.memory, "-p", f"127.0.0.1:{port}:8080", args.image, ], check=True, capture_output=True, text=True, ) samples = [] stop_sampling = threading.Event() sampler = threading.Thread(target=sample_stats, args=(container, samples, stop_sampling)) successful = 0 failures = 0 fatal_error = None started = time.monotonic() message_points_x = [] message_points_y = [] failure_time = None oom_killed = False last_error = None last_state = {} logs = "" try: wait_ready(base_url) user = request(base_url, "POST", "/api/v1/users", { "username": "memory_" + uuid.uuid4().hex[:16], "display_name": "Memory experiment", }) chat = request(base_url, "POST", "/api/v1/chats", { "title": "Memory experiment", "member_ids": [user["id"]], }, user["id"]) sampler.start() time.sleep(0.5) body = "x" * 4000 for index in range(args.max_messages): try: request( base_url, "POST", f"/api/v1/chats/{chat['id']}/messages", {"text": body}, user["id"], ) successful += 1 message_points_x.append(time.monotonic() - started) message_points_y.append(successful) except (OSError, urllib.error.HTTPError, ValueError) as exc: last_error = type(exc).__name__ + ": " + str(exc) failures += 1 failure_time = time.monotonic() - started break time.sleep(0.5) if not process_alive(container): failure_time = failure_time or (time.monotonic() - started) inspect = subprocess.run( ["docker", "inspect", "--format", "{{json .State}}", container], capture_output=True, text=True, check=False, ) state = json.loads(inspect.stdout) if inspect.stdout.strip() else {} if not state: state_result = subprocess.run( ["docker", "inspect", "--format", "{{json .State}}", container], capture_output=True, text=True, check=False, ) state = json.loads(state_result.stdout) if state_result.stdout.strip() else {} last_state = state logs_result = subprocess.run(["docker", "logs", container], capture_output=True, text=True, check=False) logs = logs_result.stdout + logs_result.stderr oom_killed = bool(state.get("OOMKilled")) if failure_time is None and failures: failure_time = time.monotonic() - started except Exception as exc: fatal_error = type(exc).__name__ + ": " + str(exc) failure_time = failure_time or (time.monotonic() - started) finally: stop_sampling.set() if sampler.is_alive(): sampler.join(timeout=2) memory_limit_mib = float(args.memory.rstrip("mM")) if args.memory.lower().endswith("m") else 0 if not oom_killed: inspect = subprocess.run( ["docker", "inspect", "--format", "{{json .State}}", container], capture_output=True, text=True, check=False, ) state = json.loads(inspect.stdout) if inspect.stdout.strip() else {} last_state = state oom_killed = bool(state.get("OOMKilled")) result = { "image": args.image, "memory_limit": args.memory, "memory_limit_mib": memory_limit_mib, "successful_messages": successful, "failed_requests": failures, "oom_killed": oom_killed, "last_error": last_error, "fatal_error": fatal_error, "container_state": last_state, "container_logs_tail": logs[-20000:], "failure_time_seconds": failure_time, "memory_samples": samples, "message_points_x": message_points_x, "message_points_y": message_points_y, "reason": "process stopped after the container memory limit was reached" if oom_killed else "request processing did not reach the memory limit", } with open(args.output, "w", encoding="utf-8") as output: json.dump(result, output, ensure_ascii=False, indent=2) make_plot(result, args.plot) print(json.dumps({key: result[key] for key in ( "memory_limit", "successful_messages", "failed_requests", "oom_killed", "failure_time_seconds", "reason", )}, ensure_ascii=False, indent=2)) subprocess.run(["docker", "rm", "-f", container], capture_output=True, check=False) if __name__ == "__main__": main()