Files
mtusi2026lab5/scripts/memory_experiment.py
T
2026-10-29 23:36:12 +03:00

272 lines
9.5 KiB
Python

import argparse
import json
import os
import subprocess
import threading
import time
import urllib.error
import urllib.request
import uuid
import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt
def request(base_url, method, path, body=None, user_id=None):
payload = json.dumps(body, ensure_ascii=False).encode() if body is not None else None
headers = {"Content-Type": "application/json"} if payload else {}
if user_id:
headers["X-User-Id"] = user_id
request = urllib.request.Request(
base_url + path,
data=payload,
headers=headers,
method=method,
)
with urllib.request.urlopen(request, timeout=3) as response:
return json.loads(response.read())
def wait_ready(base_url):
deadline = time.monotonic() + 20
while time.monotonic() < deadline:
try:
request(base_url, "GET", "/health/live")
return
except OSError:
time.sleep(0.1)
raise RuntimeError("container did not become ready")
def process_alive(container):
result = subprocess.run(
["docker", "inspect", "--format", "{{.State.Running}}", container],
capture_output=True,
text=True,
check=False,
)
return result.stdout.strip() == "true"
def sample_stats(container, samples, stop):
while not stop.is_set():
result = subprocess.run(
["docker", "stats", "--no-stream", "--format", "{{json .}}", container],
capture_output=True,
text=True,
check=False,
)
if result.returncode == 0 and result.stdout.strip():
try:
value = json.loads(result.stdout)
memory = value.get("MemUsage", "").split("/")[0].strip()
if memory == "0B" or not process_alive(container):
continue
samples.append({
"time_seconds": time.monotonic(),
"memory": memory,
"memory_mib": parse_memory(memory),
})
except (ValueError, IndexError):
pass
stop.wait(0.1)
def parse_memory(value):
units = {"B": 1 / 1024 / 1024, "KiB": 1 / 1024, "MiB": 1, "GiB": 1024}
for unit, factor in sorted(units.items(), key=lambda item: -len(item[0])):
if value.endswith(unit):
return float(value[:-len(unit)].strip()) * factor
raise ValueError(value)
def make_plot(data, output):
samples = [sample for sample in data["memory_samples"] if sample["memory_mib"] > 0]
start = samples[0]["time_seconds"] if samples else 0
x = [sample["time_seconds"] - start for sample in samples]
memory = [sample["memory_mib"] for sample in samples]
fig, memory_axis = plt.subplots(figsize=(11, 6))
memory_axis.plot(x, memory, color="tab:red", label="RSS / container memory")
memory_axis.axhline(data["memory_limit_mib"], color="black", linestyle="--", label="memory limit")
memory_axis.set_xlabel("Time, seconds")
memory_axis.set_ylabel("Memory, MiB")
memory_axis.grid(True, alpha=0.25)
message_axis = memory_axis.twinx()
message_axis.plot(
data["message_points_x"],
data["message_points_y"],
color="tab:blue",
label="successful messages",
)
message_axis.set_ylabel("Successful messages")
lines, labels = memory_axis.get_legend_handles_labels()
other_lines, other_labels = message_axis.get_legend_handles_labels()
memory_axis.legend(lines + other_lines, labels + other_labels, loc="upper left")
memory_axis.set_title("In-memory messenger: memory exhaustion point")
if data["oom_killed"] and data["failure_time_seconds"] is not None:
memory_axis.axvline(
data["failure_time_seconds"],
color="tab:purple",
linestyle=":",
label="container OOMKilled",
)
memory_axis.annotate(
"OOMKilled",
(data["failure_time_seconds"], data["memory_limit_mib"]),
xytext=(8, -18),
textcoords="offset points",
color="tab:purple",
)
fig.tight_layout()
fig.savefig(output, dpi=160)
plt.close(fig)
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--image", required=True)
parser.add_argument("--memory", default="32m")
parser.add_argument("--output", default="evidence/memory-experiment.json")
parser.add_argument("--plot", default="evidence/memory-experiment.png")
parser.add_argument("--max-messages", type=int, default=100000)
args = parser.parse_args()
container = "mtusi-memory-experiment-" + uuid.uuid4().hex[:8]
port = 18081
base_url = f"http://127.0.0.1:{port}"
subprocess.run(
[
"docker", "run", "-d", "--name", container,
"--memory", args.memory, "--memory-swap", args.memory,
"-p", f"127.0.0.1:{port}:8080", args.image,
],
check=True,
capture_output=True,
text=True,
)
samples = []
stop_sampling = threading.Event()
sampler = threading.Thread(target=sample_stats, args=(container, samples, stop_sampling))
successful = 0
failures = 0
fatal_error = None
started = time.monotonic()
message_points_x = []
message_points_y = []
failure_time = None
oom_killed = False
last_error = None
last_state = {}
logs = ""
try:
wait_ready(base_url)
user = request(base_url, "POST", "/api/v1/users", {
"username": "memory_" + uuid.uuid4().hex[:16],
"display_name": "Memory experiment",
})
chat = request(base_url, "POST", "/api/v1/chats", {
"title": "Memory experiment",
"member_ids": [user["id"]],
}, user["id"])
sampler.start()
time.sleep(0.5)
body = "x" * 4000
for index in range(args.max_messages):
try:
request(
base_url,
"POST",
f"/api/v1/chats/{chat['id']}/messages",
{"text": body},
user["id"],
)
successful += 1
message_points_x.append(time.monotonic() - started)
message_points_y.append(successful)
except (OSError, urllib.error.HTTPError, ValueError) as exc:
last_error = type(exc).__name__ + ": " + str(exc)
failures += 1
failure_time = time.monotonic() - started
break
time.sleep(0.5)
if not process_alive(container):
failure_time = failure_time or (time.monotonic() - started)
inspect = subprocess.run(
["docker", "inspect", "--format", "{{json .State}}", container],
capture_output=True,
text=True,
check=False,
)
state = json.loads(inspect.stdout) if inspect.stdout.strip() else {}
if not state:
state_result = subprocess.run(
["docker", "inspect", "--format", "{{json .State}}", container],
capture_output=True,
text=True,
check=False,
)
state = json.loads(state_result.stdout) if state_result.stdout.strip() else {}
last_state = state
logs_result = subprocess.run(["docker", "logs", container], capture_output=True, text=True, check=False)
logs = logs_result.stdout + logs_result.stderr
oom_killed = bool(state.get("OOMKilled"))
if failure_time is None and failures:
failure_time = time.monotonic() - started
except Exception as exc:
fatal_error = type(exc).__name__ + ": " + str(exc)
failure_time = failure_time or (time.monotonic() - started)
finally:
stop_sampling.set()
if sampler.is_alive():
sampler.join(timeout=2)
memory_limit_mib = float(args.memory.rstrip("mM")) if args.memory.lower().endswith("m") else 0
if not oom_killed:
inspect = subprocess.run(
["docker", "inspect", "--format", "{{json .State}}", container],
capture_output=True,
text=True,
check=False,
)
state = json.loads(inspect.stdout) if inspect.stdout.strip() else {}
last_state = state
oom_killed = bool(state.get("OOMKilled"))
result = {
"image": args.image,
"memory_limit": args.memory,
"memory_limit_mib": memory_limit_mib,
"successful_messages": successful,
"failed_requests": failures,
"oom_killed": oom_killed,
"last_error": last_error,
"fatal_error": fatal_error,
"container_state": last_state,
"container_logs_tail": logs[-20000:],
"failure_time_seconds": failure_time,
"memory_samples": samples,
"message_points_x": message_points_x,
"message_points_y": message_points_y,
"reason": "process stopped after the container memory limit was reached" if oom_killed else "request processing did not reach the memory limit",
}
with open(args.output, "w", encoding="utf-8") as output:
json.dump(result, output, ensure_ascii=False, indent=2)
make_plot(result, args.plot)
print(json.dumps({key: result[key] for key in (
"memory_limit", "successful_messages", "failed_requests", "oom_killed", "failure_time_seconds", "reason",
)}, ensure_ascii=False, indent=2))
subprocess.run(["docker", "rm", "-f", container], capture_output=True, check=False)
if __name__ == "__main__":
main()