#!/usr/bin/env python3 """Peak API memory under N concurrent ~8.7 MB uploads. Samples `docker stats` for the api containers. python3 student-tests/memory_experiment.py --concurrency 10 --rounds 3""" import argparse import json import os import re import statistics import subprocess import sys import threading import time import zlib import struct from concurrent.futures import ThreadPoolExecutor from pathlib import Path ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT / "tests")) from client import Client # noqa: E402 def png(width, height): raw = b"".join(b"\0" + os.urandom(width * 3) for _ in range(height)) def chunk(kind, body): return struct.pack(">I", len(body)) + kind + body + struct.pack(">I", zlib.crc32(kind + body) & 0xFFFFFFFF) return (b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0)) + chunk(b"IDAT", zlib.compress(raw, 1)) + chunk(b"IEND", b"")) UNITS = {"B": 1, "KiB": 1024, "MiB": 1048576, "GiB": 1073741824} def mib(text): value, unit = re.match(r"([\d.]+)\s*([A-Za-z]+)", text).groups() return float(value) * UNITS[unit] / 1048576 def sample(stop, out): while not stop.is_set(): result = subprocess.run(["docker", "stats", "--no-stream", "--format", "{{.Name}} {{.MemUsage}}"], cwd=ROOT, capture_output=True, text=True) for line in result.stdout.splitlines(): name, usage = line.split(" ", 1) if "api-" in name: out.setdefault(name, []).append(mib(usage.split("/")[0].strip())) def main(): parser = argparse.ArgumentParser() parser.add_argument("--concurrency", type=int, default=10) parser.add_argument("--rounds", type=int, default=3) args = parser.parse_args() body = png(1700, 1700) print("payload bytes:", len(body)) c = Client(timeout=60) alice = c.user("mem") chat = c.call("POST", "/api/v1/chats", 201, actor=alice, body={"title": "mem", "member_ids": [alice["id"]]}, schema="Chat")["json"] path = "/api/v1/chats/%s/attachments?filename=big.png" % chat["id"] def one(_): r = c.call("POST", path, [201, 429, 503], actor=alice, raw=body, headers={"Content-Type": "image/png"}) return r["status"], r["elapsed"] idle, peaks, statuses, times = {}, [], [], [] probe = {} stop = threading.Event() t = threading.Thread(target=sample, args=(stop, probe)); t.start(); time.sleep(3); stop.set(); t.join() idle = {k: max(v) for k, v in probe.items()} for round_no in range(args.rounds): probe = {} stop = threading.Event() t = threading.Thread(target=sample, args=(stop, probe)); t.start() with ThreadPoolExecutor(args.concurrency) as pool: results = list(pool.map(one, range(args.concurrency))) stop.set(); t.join() peak = {k: max(v) for k, v in probe.items()} peaks.append(peak) statuses += [s for s, _ in results] times += [e for _, e in results] print("round %d peak MiB per api: %s statuses=%s" % (round_no + 1, {k: round(v, 1) for k, v in peak.items()}, sorted(set(s for s, _ in results)))) summary = { "payload_mib": round(len(body) / 1048576, 2), "concurrency": args.concurrency, "rounds": args.rounds, "idle_mib": {k: round(v, 1) for k, v in idle.items()}, "peak_mib": {k: round(max(p.get(k, 0) for p in peaks), 1) for k in idle}, "container_limit_mib": 256, "statuses": {str(s): statuses.count(s) for s in sorted(set(statuses))}, "upload_seconds_median": round(statistics.median(times), 2), "upload_seconds_max": round(max(times), 2), } print(json.dumps(summary, indent=2)) main()