337 lines
9.9 KiB
Python
337 lines
9.9 KiB
Python
from __future__ import annotations
|
|
|
|
import base64
|
|
from collections import Counter
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
import csv
|
|
import hashlib
|
|
import json
|
|
from pathlib import Path
|
|
import random
|
|
import secrets
|
|
import struct
|
|
import subprocess
|
|
import threading
|
|
import time
|
|
import urllib.error
|
|
import urllib.request
|
|
import zlib
|
|
|
|
|
|
BASE = "http://127.0.0.1:18082/api/v1"
|
|
COMPOSE = [
|
|
"docker", "compose",
|
|
"-f", "compose.yaml",
|
|
"-f", "compose.s3.yaml",
|
|
"--env-file", ".env",
|
|
]
|
|
token = None
|
|
|
|
|
|
def api(method, path, *, payload=None, headers=None):
|
|
headers = dict(headers or {})
|
|
if token and "Authorization" not in headers:
|
|
headers["Authorization"] = "Bearer " + token
|
|
data = None
|
|
if payload is not None:
|
|
data = json.dumps(payload).encode()
|
|
headers["Content-Type"] = "application/json"
|
|
request = urllib.request.Request(
|
|
BASE + path, method=method, headers=headers, data=data
|
|
)
|
|
with urllib.request.urlopen(request, timeout=30) as response:
|
|
return json.loads(response.read())
|
|
|
|
|
|
def chunk(kind, data):
|
|
return (
|
|
struct.pack(">I", len(data))
|
|
+ kind + data
|
|
+ struct.pack(">I", zlib.crc32(kind + data) & 0xffffffff)
|
|
)
|
|
|
|
|
|
print("Generating PNG 6000x6000...", flush=True)
|
|
rng = random.Random(44)
|
|
compressor = zlib.compressobj(level=1)
|
|
compressed = bytearray()
|
|
|
|
for row in range(6000):
|
|
pixels = rng.randbytes(18000) if row < 450 else bytes(18000)
|
|
compressed.extend(compressor.compress(b"\x00" + pixels))
|
|
compressed.extend(compressor.flush())
|
|
|
|
image = (
|
|
b"\x89PNG\r\n\x1a\n"
|
|
+ chunk(b"IHDR", struct.pack(">IIBBBBB", 6000, 6000, 8, 2, 0, 0, 0))
|
|
+ chunk(b"IDAT", bytes(compressed))
|
|
+ chunk(b"IEND", b"")
|
|
)
|
|
del compressed
|
|
assert len(image) <= 10485760
|
|
digest = hashlib.sha256(image).hexdigest()
|
|
print(f"Fixture: {len(image)} bytes, 36000000 pixels", flush=True)
|
|
|
|
username = "memory_" + secrets.token_hex(6)
|
|
password = "Lab4-" + secrets.token_hex(16)
|
|
user = api(
|
|
"POST", "/users",
|
|
payload={
|
|
"username": username,
|
|
"display_name": "Upload memory experiment",
|
|
"password": password,
|
|
},
|
|
)
|
|
basic = base64.b64encode(f"{username}:{password}".encode()).decode()
|
|
session = api(
|
|
"POST", "/auth/sessions",
|
|
headers={"Authorization": "Basic " + basic},
|
|
)
|
|
token = session["token"]
|
|
chat = api(
|
|
"POST", "/chats",
|
|
payload={
|
|
"title": "Upload memory experiment",
|
|
"member_ids": [user["id"]],
|
|
},
|
|
)
|
|
|
|
containers = subprocess.check_output(
|
|
COMPOSE + ["ps", "-q", "api"], text=True
|
|
).split()
|
|
assert len(containers) >= 2, "Need two running API replicas"
|
|
|
|
|
|
def container_states():
|
|
result = {}
|
|
for container in containers:
|
|
text = subprocess.check_output(
|
|
[
|
|
"docker", "inspect", "--format",
|
|
"{{json .State}}|{{.RestartCount}}", container,
|
|
],
|
|
text=True,
|
|
)
|
|
raw, restart_count = text.strip().rsplit("|", 1)
|
|
state = json.loads(raw)
|
|
result[container[:12]] = {
|
|
"started_at": state["StartedAt"],
|
|
"oom_killed": state["OOMKilled"],
|
|
"status": state["Status"],
|
|
"restart_count": int(restart_count),
|
|
}
|
|
return result
|
|
|
|
|
|
SAMPLER = r'''
|
|
import json
|
|
from pathlib import Path
|
|
import select
|
|
import sys
|
|
|
|
while True:
|
|
row = {}
|
|
for line in Path("/proc/1/status").read_text().splitlines():
|
|
name, _, value = line.partition(":")
|
|
if name in ("VmRSS", "VmHWM", "VmSwap"):
|
|
row[name + "_kib"] = int(value.split()[0])
|
|
|
|
root = Path("/sys/fs/cgroup")
|
|
for name in ("memory.current", "memory.peak", "memory.max"):
|
|
row[name] = int((root / name).read_text().strip())
|
|
|
|
for line in (root / "memory.events").read_text().splitlines():
|
|
name, value = line.split()
|
|
row["events." + name] = int(value)
|
|
|
|
print(json.dumps(row), flush=True)
|
|
ready, _, _ = select.select([sys.stdin], [], [], 0.1)
|
|
if ready and not sys.stdin.readline():
|
|
break
|
|
'''
|
|
|
|
samples = []
|
|
monitor_errors = []
|
|
monitors = []
|
|
readers = []
|
|
phase = {"name": "baseline"}
|
|
origin = time.monotonic()
|
|
before = container_states()
|
|
|
|
|
|
def read_samples(process, container):
|
|
try:
|
|
for line in process.stdout:
|
|
row = json.loads(line)
|
|
row["container"] = container[:12]
|
|
row["time_s"] = round(time.monotonic() - origin, 4)
|
|
row["phase"] = phase["name"]
|
|
samples.append(row)
|
|
except Exception as error:
|
|
monitor_errors.append(type(error).__name__)
|
|
|
|
|
|
for container in containers:
|
|
process = subprocess.Popen(
|
|
["docker", "exec", "-i", container, "python", "-u", "-c", SAMPLER],
|
|
stdin=subprocess.PIPE,
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE,
|
|
text=True,
|
|
)
|
|
monitors.append(process)
|
|
reader = threading.Thread(
|
|
target=read_samples, args=(process, container), daemon=True
|
|
)
|
|
reader.start()
|
|
readers.append(reader)
|
|
|
|
rounds = []
|
|
|
|
|
|
def run_round(label, concurrency):
|
|
barrier = threading.Barrier(concurrency + 1)
|
|
lock = threading.Lock()
|
|
activity = {"current": 0, "peak": 0}
|
|
|
|
def worker(number):
|
|
barrier.wait()
|
|
with lock:
|
|
activity["current"] += 1
|
|
activity["peak"] = max(
|
|
activity["peak"], activity["current"]
|
|
)
|
|
started = time.monotonic()
|
|
result = {"number": number}
|
|
try:
|
|
request = urllib.request.Request(
|
|
BASE + f"/chats/{chat['id']}/attachments"
|
|
+ f"?filename={label}-{number}.png",
|
|
method="POST",
|
|
data=image,
|
|
headers={
|
|
"Authorization": "Bearer " + token,
|
|
"Content-Type": "image/png",
|
|
},
|
|
)
|
|
try:
|
|
response = urllib.request.urlopen(request, timeout=60)
|
|
except urllib.error.HTTPError as error:
|
|
response = error
|
|
|
|
with response:
|
|
body = response.read()
|
|
result["status"] = response.code
|
|
result["instance"] = response.headers.get("X-Instance-Id")
|
|
if response.code == 201:
|
|
metadata = json.loads(body)
|
|
result["valid_metadata"] = (
|
|
metadata["sha256"] == digest
|
|
and metadata["size_bytes"] == len(image)
|
|
and metadata["status"] == "ready"
|
|
)
|
|
except Exception as error:
|
|
result["status"] = "transport_error"
|
|
result["error_type"] = type(error).__name__
|
|
finally:
|
|
result["seconds"] = round(time.monotonic() - started, 4)
|
|
with lock:
|
|
activity["current"] -= 1
|
|
return result
|
|
|
|
phase["name"] = label
|
|
with ThreadPoolExecutor(max_workers=concurrency) as executor:
|
|
futures = [executor.submit(worker, n) for n in range(concurrency)]
|
|
started = time.monotonic()
|
|
barrier.wait()
|
|
results = [future.result() for future in futures]
|
|
elapsed = time.monotonic() - started
|
|
|
|
summary = {
|
|
"label": label,
|
|
"clients": concurrency,
|
|
"peak_client_in_flight": activity["peak"],
|
|
"elapsed_seconds": round(elapsed, 4),
|
|
"statuses": dict(Counter(str(row["status"]) for row in results)),
|
|
"requests": results,
|
|
}
|
|
rounds.append(summary)
|
|
print(json.dumps({
|
|
key: value for key, value in summary.items()
|
|
if key != "requests"
|
|
}), flush=True)
|
|
|
|
phase["name"] = label + "_cooldown"
|
|
time.sleep(2)
|
|
|
|
|
|
try:
|
|
time.sleep(1)
|
|
run_round("single", 1)
|
|
run_round("concurrent_1", 10)
|
|
run_round("concurrent_2", 10)
|
|
finally:
|
|
phase["name"] = "final"
|
|
time.sleep(1)
|
|
for process in monitors:
|
|
process.stdin.close()
|
|
for process in monitors:
|
|
try:
|
|
process.wait(timeout=10)
|
|
except subprocess.TimeoutExpired:
|
|
process.terminate()
|
|
monitor_errors.append("sampler_stop_timeout")
|
|
if process.returncode != 0:
|
|
monitor_errors.append("sampler_nonzero_exit")
|
|
for reader in readers:
|
|
reader.join(timeout=5)
|
|
|
|
after = container_states()
|
|
report = {
|
|
"fixture_bytes": len(image),
|
|
"width": 6000,
|
|
"height": 6000,
|
|
"fixture_sha256": digest,
|
|
"before": before,
|
|
"after": after,
|
|
"rounds": rounds,
|
|
"monitor_errors": monitor_errors,
|
|
"memory_summary": {},
|
|
}
|
|
|
|
for container in containers:
|
|
key = container[:12]
|
|
rows = [row for row in samples if row["container"] == key]
|
|
assert rows, f"No memory samples for {key}"
|
|
report["memory_summary"][key] = {
|
|
"samples": len(rows),
|
|
"initial_cgroup_peak_bytes": rows[0]["memory.peak"],
|
|
"final_cgroup_peak_bytes": rows[-1]["memory.peak"],
|
|
"sampled_max_cgroup_bytes": max(r["memory.current"] for r in rows),
|
|
"sampled_max_rss_kib": max(r["VmRSS_kib"] for r in rows),
|
|
"initial_hwm_kib": rows[0]["VmHWM_kib"],
|
|
"final_hwm_kib": rows[-1]["VmHWM_kib"],
|
|
"oom_delta": rows[-1]["events.oom"] - rows[0]["events.oom"],
|
|
"oom_kill_delta": (
|
|
rows[-1]["events.oom_kill"] - rows[0]["events.oom_kill"]
|
|
),
|
|
}
|
|
|
|
directory = Path("evidence")
|
|
directory.mkdir(exist_ok=True)
|
|
(directory / "upload_memory.json").write_text(
|
|
json.dumps(report, indent=2) + "\n"
|
|
)
|
|
fields = sorted({key for row in samples for key in row})
|
|
with (directory / "upload_memory.csv").open("w", newline="") as stream:
|
|
writer = csv.DictWriter(stream, fieldnames=fields, lineterminator="\n")
|
|
writer.writeheader()
|
|
writer.writerows(samples)
|
|
|
|
print("\nMEMORY SUMMARY", flush=True)
|
|
print(json.dumps(report["memory_summary"], indent=2), flush=True)
|
|
print("Container states unchanged:", before == after, flush=True)
|
|
print("Monitor errors:", monitor_errors, flush=True)
|
|
print("Saved: evidence/upload_memory.json and evidence/upload_memory.csv")
|
|
print("EXPERIMENT COMPLETE: inspect statuses and memory before conclusions")
|