Files
mtusi2026lab4/scripts/memory_experiment.py
T

172 lines
5.7 KiB
Python

"""Ten concurrent uploads at two sizes. Records container RSS, not a universal capacity claim."""
import json
import random
import struct
import subprocess
import sys
import threading
import time
import urllib.request
import zlib
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
from client import Client # noqa: E402
def chunk(tag: bytes, data: bytes) -> bytes:
crc = zlib.crc32(tag + data) & 0xFFFFFFFF
return struct.pack(">I", len(data)) + tag + data + struct.pack(">I", crc)
def make_png(width: int, height: int, seed: int) -> bytes:
rng = random.Random(seed)
raw = bytearray()
for _y in range(height):
raw.append(0)
raw.extend(rng.randbytes(width * 3))
compressor = zlib.compressobj(level=0)
payload = compressor.compress(bytes(raw)) + compressor.flush()
ihdr = struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0)
return b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", ihdr) + chunk(b"IDAT", payload) + chunk(b"IEND", b"")
def parse_bytes(text: str) -> int:
"""docker stats MemUsage looks like '12.3MiB / 768MiB'."""
value = text.split("/")[0].strip()
number = ""
unit = ""
for char in value:
if char.isdigit() or char == ".":
number += char
elif char != " ":
unit += char
scale = {"B": 1, "KiB": 1024, "MiB": 1024**2, "GiB": 1024**3, "kB": 1000, "MB": 1000**2, "GB": 1000**3}
return int(float(number) * scale[unit])
def sample(stop: threading.Event, peaks: dict) -> None:
while not stop.is_set():
try:
raw = subprocess.check_output(
["docker", "stats", "--no-stream", "--format", "{{.Name}}\t{{.MemUsage}}"],
text=True,
)
except subprocess.CalledProcessError:
time.sleep(0.3)
continue
for line in raw.splitlines():
name, usage = line.split("\t", 1)
if name not in {"mtusi-api-1", "mtusi-api-2"}:
continue
peaks[name] = max(peaks.get(name, 0), parse_bytes(usage))
time.sleep(0.25)
def vmhwm() -> dict:
result = {}
for name in ("mtusi-api-1", "mtusi-api-2"):
text = subprocess.check_output(["docker", "exec", name, "sh", "-c", "grep VmHWM /proc/1/status"], text=True)
result[name] = int(text.split()[1])
return result
def wave(client: Client, actor, chat, payload: bytes, label: str) -> dict:
stop = threading.Event()
peaks = {}
thread = threading.Thread(target=sample, args=(stop, peaks))
before = vmhwm()
thread.start()
errors = []
def once(index):
local = Client("http://127.0.0.1:8080", timeout=60)
path = f"/api/v1/chats/{chat['id']}/attachments?filename={label}-{index}.png"
try:
response = local.call(
"POST",
path,
201,
actor=actor,
raw=payload,
headers={"Content-Type": "image/png"},
schema="Attachment",
)
return response["json"]["id"]
except Exception as exc:
errors.append(str(exc))
return None
started = time.perf_counter()
from concurrent.futures import ThreadPoolExecutor
with ThreadPoolExecutor(max_workers=10) as pool:
ids = list(pool.map(once, range(10)))
elapsed = time.perf_counter() - started
time.sleep(0.6)
stop.set()
thread.join()
after = vmhwm()
return {
"label": label,
"file_bytes": len(payload),
"concurrency": 10,
"successes": sum(1 for item in ids if item),
"errors": errors[:5],
"elapsed_seconds": round(elapsed, 3),
"docker_stats_peak_bytes": peaks,
"vmhwm_kib_before": before,
"vmhwm_kib_after": after,
}
def restart_apis() -> None:
subprocess.check_call(["docker", "compose", "--env-file", ".env", "restart", "api-1", "api-2"], cwd=ROOT)
deadline = time.time() + 90
while time.time() < deadline:
try:
urllib.request.urlopen("http://127.0.0.1:8080/health/ready", timeout=2).read()
return
except Exception:
time.sleep(1)
raise SystemExit("API did not become ready after restart")
def main() -> None:
client = Client("http://127.0.0.1:8080", timeout=30)
alice = client.user("mem_alice")
bob = client.user("mem_bob")
chat = client.call(
"POST",
"/api/v1/chats",
201,
actor=alice,
body={"title": "Память", "member_ids": [alice["id"], bob["id"]]},
schema="Chat",
)["json"]
small = make_png(64, 64, 1)
large = make_png(1200, 900, 2)
restart_apis()
first = wave(client, alice, chat, small, "small")
restart_apis()
second = wave(client, alice, chat, large, "large")
report = {
"hypothesis": "A single upload is capped at 10 MiB and does not decode bitmaps above 1_000_000 pixels; ten concurrent uploads stay inside the 768 MiB container limit.",
"limits": {"container_memory": "768MiB", "max_upload_bytes": 10485760, "full_decode_pixels": 1000000},
"waves": [first, second],
"note": "docker stats includes page cache. VmHWM is the process high-water RSS in KiB after a fresh API restart, so the two waves are comparable.",
}
dest = ROOT / "evidence" / "memory.json"
dest.parent.mkdir(exist_ok=True)
dest.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
print(json.dumps(report, ensure_ascii=False, indent=2))
if first["successes"] != 10 or second["successes"] != 10:
raise SystemExit("not all concurrent uploads succeeded")
print("PASS: ten concurrent uploads at two sizes")
if __name__ == "__main__":
main()