lab1
This commit is contained in:
@@ -0,0 +1,271 @@
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import threading
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
import uuid
|
||||
|
||||
import matplotlib
|
||||
|
||||
matplotlib.use("Agg")
|
||||
import matplotlib.pyplot as plt
|
||||
|
||||
|
||||
def request(base_url, method, path, body=None, user_id=None):
|
||||
payload = json.dumps(body, ensure_ascii=False).encode() if body is not None else None
|
||||
headers = {"Content-Type": "application/json"} if payload else {}
|
||||
if user_id:
|
||||
headers["X-User-Id"] = user_id
|
||||
request = urllib.request.Request(
|
||||
base_url + path,
|
||||
data=payload,
|
||||
headers=headers,
|
||||
method=method,
|
||||
)
|
||||
with urllib.request.urlopen(request, timeout=3) as response:
|
||||
return json.loads(response.read())
|
||||
|
||||
|
||||
def wait_ready(base_url):
|
||||
deadline = time.monotonic() + 20
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
request(base_url, "GET", "/health/live")
|
||||
return
|
||||
except OSError:
|
||||
time.sleep(0.1)
|
||||
raise RuntimeError("container did not become ready")
|
||||
|
||||
|
||||
def process_alive(container):
|
||||
result = subprocess.run(
|
||||
["docker", "inspect", "--format", "{{.State.Running}}", container],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
return result.stdout.strip() == "true"
|
||||
|
||||
|
||||
def sample_stats(container, samples, stop):
|
||||
while not stop.is_set():
|
||||
result = subprocess.run(
|
||||
["docker", "stats", "--no-stream", "--format", "{{json .}}", container],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
if result.returncode == 0 and result.stdout.strip():
|
||||
try:
|
||||
value = json.loads(result.stdout)
|
||||
memory = value.get("MemUsage", "").split("/")[0].strip()
|
||||
if memory == "0B" or not process_alive(container):
|
||||
continue
|
||||
samples.append({
|
||||
"time_seconds": time.monotonic(),
|
||||
"memory": memory,
|
||||
"memory_mib": parse_memory(memory),
|
||||
})
|
||||
except (ValueError, IndexError):
|
||||
pass
|
||||
stop.wait(0.1)
|
||||
|
||||
|
||||
def parse_memory(value):
|
||||
units = {"B": 1 / 1024 / 1024, "KiB": 1 / 1024, "MiB": 1, "GiB": 1024}
|
||||
for unit, factor in sorted(units.items(), key=lambda item: -len(item[0])):
|
||||
if value.endswith(unit):
|
||||
return float(value[:-len(unit)].strip()) * factor
|
||||
raise ValueError(value)
|
||||
|
||||
|
||||
def make_plot(data, output):
|
||||
samples = [sample for sample in data["memory_samples"] if sample["memory_mib"] > 0]
|
||||
start = samples[0]["time_seconds"] if samples else 0
|
||||
x = [sample["time_seconds"] - start for sample in samples]
|
||||
memory = [sample["memory_mib"] for sample in samples]
|
||||
|
||||
fig, memory_axis = plt.subplots(figsize=(11, 6))
|
||||
memory_axis.plot(x, memory, color="tab:red", label="RSS / container memory")
|
||||
memory_axis.axhline(data["memory_limit_mib"], color="black", linestyle="--", label="memory limit")
|
||||
memory_axis.set_xlabel("Time, seconds")
|
||||
memory_axis.set_ylabel("Memory, MiB")
|
||||
memory_axis.grid(True, alpha=0.25)
|
||||
|
||||
message_axis = memory_axis.twinx()
|
||||
message_axis.plot(
|
||||
data["message_points_x"],
|
||||
data["message_points_y"],
|
||||
color="tab:blue",
|
||||
label="successful messages",
|
||||
)
|
||||
message_axis.set_ylabel("Successful messages")
|
||||
|
||||
lines, labels = memory_axis.get_legend_handles_labels()
|
||||
other_lines, other_labels = message_axis.get_legend_handles_labels()
|
||||
memory_axis.legend(lines + other_lines, labels + other_labels, loc="upper left")
|
||||
memory_axis.set_title("In-memory messenger: memory exhaustion point")
|
||||
if data["oom_killed"] and data["failure_time_seconds"] is not None:
|
||||
memory_axis.axvline(
|
||||
data["failure_time_seconds"],
|
||||
color="tab:purple",
|
||||
linestyle=":",
|
||||
label="container OOMKilled",
|
||||
)
|
||||
memory_axis.annotate(
|
||||
"OOMKilled",
|
||||
(data["failure_time_seconds"], data["memory_limit_mib"]),
|
||||
xytext=(8, -18),
|
||||
textcoords="offset points",
|
||||
color="tab:purple",
|
||||
)
|
||||
fig.tight_layout()
|
||||
fig.savefig(output, dpi=160)
|
||||
plt.close(fig)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--image", required=True)
|
||||
parser.add_argument("--memory", default="32m")
|
||||
parser.add_argument("--output", default="evidence/memory-experiment.json")
|
||||
parser.add_argument("--plot", default="evidence/memory-experiment.png")
|
||||
parser.add_argument("--max-messages", type=int, default=100000)
|
||||
args = parser.parse_args()
|
||||
|
||||
container = "mtusi-memory-experiment-" + uuid.uuid4().hex[:8]
|
||||
port = 18081
|
||||
base_url = f"http://127.0.0.1:{port}"
|
||||
subprocess.run(
|
||||
[
|
||||
"docker", "run", "-d", "--name", container,
|
||||
"--memory", args.memory, "--memory-swap", args.memory,
|
||||
"-p", f"127.0.0.1:{port}:8080", args.image,
|
||||
],
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
|
||||
samples = []
|
||||
stop_sampling = threading.Event()
|
||||
sampler = threading.Thread(target=sample_stats, args=(container, samples, stop_sampling))
|
||||
successful = 0
|
||||
failures = 0
|
||||
fatal_error = None
|
||||
started = time.monotonic()
|
||||
message_points_x = []
|
||||
message_points_y = []
|
||||
failure_time = None
|
||||
oom_killed = False
|
||||
last_error = None
|
||||
last_state = {}
|
||||
logs = ""
|
||||
|
||||
try:
|
||||
wait_ready(base_url)
|
||||
user = request(base_url, "POST", "/api/v1/users", {
|
||||
"username": "memory_" + uuid.uuid4().hex[:16],
|
||||
"display_name": "Memory experiment",
|
||||
})
|
||||
chat = request(base_url, "POST", "/api/v1/chats", {
|
||||
"title": "Memory experiment",
|
||||
"member_ids": [user["id"]],
|
||||
}, user["id"])
|
||||
sampler.start()
|
||||
time.sleep(0.5)
|
||||
|
||||
body = "x" * 4000
|
||||
for index in range(args.max_messages):
|
||||
try:
|
||||
request(
|
||||
base_url,
|
||||
"POST",
|
||||
f"/api/v1/chats/{chat['id']}/messages",
|
||||
{"text": body},
|
||||
user["id"],
|
||||
)
|
||||
successful += 1
|
||||
message_points_x.append(time.monotonic() - started)
|
||||
message_points_y.append(successful)
|
||||
except (OSError, urllib.error.HTTPError, ValueError) as exc:
|
||||
last_error = type(exc).__name__ + ": " + str(exc)
|
||||
failures += 1
|
||||
failure_time = time.monotonic() - started
|
||||
break
|
||||
|
||||
time.sleep(0.5)
|
||||
if not process_alive(container):
|
||||
failure_time = failure_time or (time.monotonic() - started)
|
||||
inspect = subprocess.run(
|
||||
["docker", "inspect", "--format", "{{json .State}}", container],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
state = json.loads(inspect.stdout) if inspect.stdout.strip() else {}
|
||||
if not state:
|
||||
state_result = subprocess.run(
|
||||
["docker", "inspect", "--format", "{{json .State}}", container],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
state = json.loads(state_result.stdout) if state_result.stdout.strip() else {}
|
||||
last_state = state
|
||||
logs_result = subprocess.run(["docker", "logs", container], capture_output=True, text=True, check=False)
|
||||
logs = logs_result.stdout + logs_result.stderr
|
||||
oom_killed = bool(state.get("OOMKilled"))
|
||||
if failure_time is None and failures:
|
||||
failure_time = time.monotonic() - started
|
||||
except Exception as exc:
|
||||
fatal_error = type(exc).__name__ + ": " + str(exc)
|
||||
failure_time = failure_time or (time.monotonic() - started)
|
||||
finally:
|
||||
stop_sampling.set()
|
||||
if sampler.is_alive():
|
||||
sampler.join(timeout=2)
|
||||
|
||||
memory_limit_mib = float(args.memory.rstrip("mM")) if args.memory.lower().endswith("m") else 0
|
||||
if not oom_killed:
|
||||
inspect = subprocess.run(
|
||||
["docker", "inspect", "--format", "{{json .State}}", container],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
state = json.loads(inspect.stdout) if inspect.stdout.strip() else {}
|
||||
last_state = state
|
||||
oom_killed = bool(state.get("OOMKilled"))
|
||||
|
||||
result = {
|
||||
"image": args.image,
|
||||
"memory_limit": args.memory,
|
||||
"memory_limit_mib": memory_limit_mib,
|
||||
"successful_messages": successful,
|
||||
"failed_requests": failures,
|
||||
"oom_killed": oom_killed,
|
||||
"last_error": last_error,
|
||||
"fatal_error": fatal_error,
|
||||
"container_state": last_state,
|
||||
"container_logs_tail": logs[-20000:],
|
||||
"failure_time_seconds": failure_time,
|
||||
"memory_samples": samples,
|
||||
"message_points_x": message_points_x,
|
||||
"message_points_y": message_points_y,
|
||||
"reason": "process stopped after the container memory limit was reached" if oom_killed else "request processing did not reach the memory limit",
|
||||
}
|
||||
with open(args.output, "w", encoding="utf-8") as output:
|
||||
json.dump(result, output, ensure_ascii=False, indent=2)
|
||||
make_plot(result, args.plot)
|
||||
print(json.dumps({key: result[key] for key in (
|
||||
"memory_limit", "successful_messages", "failed_requests", "oom_killed", "failure_time_seconds", "reason",
|
||||
)}, ensure_ascii=False, indent=2))
|
||||
subprocess.run(["docker", "rm", "-f", container], capture_output=True, check=False)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user