forked from mtusi2026/mtusi2026lab4
172 lines
5.7 KiB
Python
172 lines
5.7 KiB
Python
"""Ten concurrent uploads at two sizes. Records container RSS, not a universal capacity claim."""
|
|
|
|
import json
|
|
import random
|
|
import struct
|
|
import subprocess
|
|
import sys
|
|
import threading
|
|
import time
|
|
import urllib.request
|
|
import zlib
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT / "tests"))
|
|
from client import Client # noqa: E402
|
|
|
|
|
|
def chunk(tag: bytes, data: bytes) -> bytes:
|
|
crc = zlib.crc32(tag + data) & 0xFFFFFFFF
|
|
return struct.pack(">I", len(data)) + tag + data + struct.pack(">I", crc)
|
|
|
|
|
|
def make_png(width: int, height: int, seed: int) -> bytes:
|
|
rng = random.Random(seed)
|
|
raw = bytearray()
|
|
for _y in range(height):
|
|
raw.append(0)
|
|
raw.extend(rng.randbytes(width * 3))
|
|
compressor = zlib.compressobj(level=0)
|
|
payload = compressor.compress(bytes(raw)) + compressor.flush()
|
|
ihdr = struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0)
|
|
return b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", ihdr) + chunk(b"IDAT", payload) + chunk(b"IEND", b"")
|
|
|
|
|
|
def parse_bytes(text: str) -> int:
|
|
"""docker stats MemUsage looks like '12.3MiB / 768MiB'."""
|
|
value = text.split("/")[0].strip()
|
|
number = ""
|
|
unit = ""
|
|
for char in value:
|
|
if char.isdigit() or char == ".":
|
|
number += char
|
|
elif char != " ":
|
|
unit += char
|
|
scale = {"B": 1, "KiB": 1024, "MiB": 1024**2, "GiB": 1024**3, "kB": 1000, "MB": 1000**2, "GB": 1000**3}
|
|
return int(float(number) * scale[unit])
|
|
|
|
|
|
def sample(stop: threading.Event, peaks: dict) -> None:
|
|
while not stop.is_set():
|
|
try:
|
|
raw = subprocess.check_output(
|
|
["docker", "stats", "--no-stream", "--format", "{{.Name}}\t{{.MemUsage}}"],
|
|
text=True,
|
|
)
|
|
except subprocess.CalledProcessError:
|
|
time.sleep(0.3)
|
|
continue
|
|
for line in raw.splitlines():
|
|
name, usage = line.split("\t", 1)
|
|
if name not in {"mtusi-api-1", "mtusi-api-2"}:
|
|
continue
|
|
peaks[name] = max(peaks.get(name, 0), parse_bytes(usage))
|
|
time.sleep(0.25)
|
|
|
|
|
|
def vmhwm() -> dict:
|
|
result = {}
|
|
for name in ("mtusi-api-1", "mtusi-api-2"):
|
|
text = subprocess.check_output(["docker", "exec", name, "sh", "-c", "grep VmHWM /proc/1/status"], text=True)
|
|
result[name] = int(text.split()[1])
|
|
return result
|
|
|
|
|
|
def wave(client: Client, actor, chat, payload: bytes, label: str) -> dict:
|
|
stop = threading.Event()
|
|
peaks = {}
|
|
thread = threading.Thread(target=sample, args=(stop, peaks))
|
|
before = vmhwm()
|
|
thread.start()
|
|
errors = []
|
|
|
|
def once(index):
|
|
local = Client("http://127.0.0.1:8080", timeout=60)
|
|
path = f"/api/v1/chats/{chat['id']}/attachments?filename={label}-{index}.png"
|
|
try:
|
|
response = local.call(
|
|
"POST",
|
|
path,
|
|
201,
|
|
actor=actor,
|
|
raw=payload,
|
|
headers={"Content-Type": "image/png"},
|
|
schema="Attachment",
|
|
)
|
|
return response["json"]["id"]
|
|
except Exception as exc:
|
|
errors.append(str(exc))
|
|
return None
|
|
|
|
started = time.perf_counter()
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
|
|
with ThreadPoolExecutor(max_workers=10) as pool:
|
|
ids = list(pool.map(once, range(10)))
|
|
elapsed = time.perf_counter() - started
|
|
time.sleep(0.6)
|
|
stop.set()
|
|
thread.join()
|
|
after = vmhwm()
|
|
return {
|
|
"label": label,
|
|
"file_bytes": len(payload),
|
|
"concurrency": 10,
|
|
"successes": sum(1 for item in ids if item),
|
|
"errors": errors[:5],
|
|
"elapsed_seconds": round(elapsed, 3),
|
|
"docker_stats_peak_bytes": peaks,
|
|
"vmhwm_kib_before": before,
|
|
"vmhwm_kib_after": after,
|
|
}
|
|
|
|
|
|
def restart_apis() -> None:
|
|
subprocess.check_call(["docker", "compose", "--env-file", ".env", "restart", "api-1", "api-2"], cwd=ROOT)
|
|
deadline = time.time() + 90
|
|
while time.time() < deadline:
|
|
try:
|
|
urllib.request.urlopen("http://127.0.0.1:8080/health/ready", timeout=2).read()
|
|
return
|
|
except Exception:
|
|
time.sleep(1)
|
|
raise SystemExit("API did not become ready after restart")
|
|
|
|
|
|
def main() -> None:
|
|
client = Client("http://127.0.0.1:8080", timeout=30)
|
|
alice = client.user("mem_alice")
|
|
bob = client.user("mem_bob")
|
|
chat = client.call(
|
|
"POST",
|
|
"/api/v1/chats",
|
|
201,
|
|
actor=alice,
|
|
body={"title": "Память", "member_ids": [alice["id"], bob["id"]]},
|
|
schema="Chat",
|
|
)["json"]
|
|
small = make_png(64, 64, 1)
|
|
large = make_png(1200, 900, 2)
|
|
restart_apis()
|
|
first = wave(client, alice, chat, small, "small")
|
|
restart_apis()
|
|
second = wave(client, alice, chat, large, "large")
|
|
report = {
|
|
"hypothesis": "A single upload is capped at 10 MiB and does not decode bitmaps above 1_000_000 pixels; ten concurrent uploads stay inside the 768 MiB container limit.",
|
|
"limits": {"container_memory": "768MiB", "max_upload_bytes": 10485760, "full_decode_pixels": 1000000},
|
|
"waves": [first, second],
|
|
"note": "docker stats includes page cache. VmHWM is the process high-water RSS in KiB after a fresh API restart, so the two waves are comparable.",
|
|
}
|
|
dest = ROOT / "evidence" / "memory.json"
|
|
dest.parent.mkdir(exist_ok=True)
|
|
dest.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
|
print(json.dumps(report, ensure_ascii=False, indent=2))
|
|
if first["successes"] != 10 or second["successes"] != 10:
|
|
raise SystemExit("not all concurrent uploads succeeded")
|
|
print("PASS: ten concurrent uploads at two sizes")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|