#!/usr/bin/env python3 """Level 4 experiment: peak memory of the API replicas under >= 10 simultaneous uploads. Hypothesis: the memory of one replica is bounded by UPLOAD_CONCURRENCY x MAX_UPLOAD_BYTES (bodies waiting for a slot stay in TCP buffers, not in the process) + one image check at a time (IMAGE_VERIFY_CONCURRENCY), independent of how many clients upload at once. Method (repeatable, the stand must be up): * for every scenario the api replicas are RECREATED with the scenario's settings, so the cgroup counter memory.peak starts from an idle process (it never decreases during a container's life); * idle memory.current of each replica is read; then N clients upload at the same moment through the LB (a barrier releases them together), each a different file; * during the run memory.current of every replica is sampled every 50 ms inside the container; afterwards memory.peak (cgroup v2: all processes of the container incl. djpeg) and VmHWM of PID 1 (the Python server alone) are read; * the normal configuration is restored at the end. python bench/upload_memory.py --output evidence/memory """ from __future__ import annotations import argparse import io import json import os import platform import statistics import subprocess import sys import threading import time import uuid from pathlib import Path import httpx2 as httpx from PIL import Image ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT / "student-tests")) import imagegen # noqa: E402 BASE = os.environ.get("BASE_URL", "http://127.0.0.1:8080") PASSWORD = "Bench-password-Ω-12345" MiB = 1024 * 1024 def sh(*args: str, env: dict | None = None, timeout: float = 300) -> str: return subprocess.run(args, cwd=ROOT, capture_output=True, text=True, check=True, timeout=timeout, env={**os.environ, **(env or {})}).stdout def replicas() -> list[str]: return sh("docker", "compose", "ps", "-q", "api").split() def cg(cid: str, name: str) -> int: return int(sh("docker", "exec", cid, "cat", f"/sys/fs/cgroup/{name}").split()[0]) def memory_stat(cid: str, key: str) -> int: for line in sh("docker", "exec", cid, "cat", "/sys/fs/cgroup/memory.stat").splitlines(): name, value = line.split() if name == key: return int(value) return 0 def vmhwm(cid: str) -> int: for line in sh("docker", "exec", cid, "cat", "/proc/1/status").splitlines(): if line.startswith("VmHWM:"): return int(line.split()[1]) * 1024 return 0 def container_events() -> dict[str, str]: """restart count + OOM-kill events per container (cgroup memory.events oom_kill counter).""" out = {} for cid in sh("docker", "compose", "ps", "-q").split(): name, restarts = sh("docker", "inspect", "-f", "{{.Name}} {{.RestartCount}}", cid).split() events = dict(line.split() for line in sh("docker", "exec", cid, "cat", "/sys/fs/cgroup/memory.events").splitlines()) out[name.lstrip("/")] = f"restarts={restarts} oom_kill={events.get('oom_kill', '?')}" return out def recreate(env: dict) -> None: sh("docker", "compose", "up", "-d", "--force-recreate", "--no-deps", "--wait", "api", env=env, timeout=300) sh(sys.executable, "scripts/lbctl.py", "wait-up", str(len(replicas())), "--timeout", "90") # LB sees them UP time.sleep(3) class Sampler(threading.Thread): """memory.current of one container every 50 ms, read by a shell loop inside it.""" def __init__(self, cid: str): super().__init__(daemon=True) self.cid = cid self.values: list[int] = [] self.proc = subprocess.Popen(["docker", "exec", cid, "sh", "-c", "while :; do cat /sys/fs/cgroup/memory.current; sleep 0.05; done"], stdout=subprocess.PIPE, text=True) def run(self) -> None: for line in self.proc.stdout: if line.strip().isdigit(): self.values.append(int(line)) def stop(self) -> None: self.proc.kill() def actor(client: httpx.Client) -> tuple[str, str]: name = "mem_" + uuid.uuid4().hex[:10] client.post("/api/v1/users", json={"username": name, "display_name": "bench", "password": PASSWORD}).raise_for_status() import base64 basic = base64.b64encode(f"{name}:{PASSWORD}".encode()).decode() token = client.post("/api/v1/auth/sessions", headers={"Authorization": "Basic " + basic}).json()["token"] me = client.get("/api/v1/users/me", headers={"Authorization": "Bearer " + token}).json() chat = client.post("/api/v1/chats", headers={"Authorization": "Bearer " + token}, json={"title": "mem", "member_ids": [me["id"]]}).json() return token, chat["id"] def big_jpeg(width: int, height: int, seed: int) -> bytes: """Large progressive JPEG (smooth content keeps it < 10 MiB): the costliest file to check.""" image = Image.linear_gradient("L").resize((width, height)).convert("RGB") image.putpixel((seed % width, 0), (seed % 255, 0, 0)) out = io.BytesIO() image.save(out, "JPEG", quality=90, progressive=True, subsampling=0) return out.getvalue() def run_scenario(name: str, env: dict, files: list[tuple[bytes, str]]) -> dict: recreate(env) ids = replicas() idle = {cid: cg(cid, "memory.current") for cid in ids} client = httpx.Client(base_url=BASE, timeout=120) token, chat_id = actor(client) barrier = threading.Barrier(len(files)) results: list[tuple[int, float, str]] = [None] * len(files) # type: ignore[list-item] def upload(i: int, data: bytes, ctype: str) -> None: c = httpx.Client(base_url=BASE, timeout=120) barrier.wait() started = time.perf_counter() r = c.post(f"/api/v1/chats/{chat_id}/attachments?filename=f{i}", content=data, headers={"Authorization": "Bearer " + token, "Content-Type": ctype}) results[i] = (r.status_code, time.perf_counter() - started, r.headers.get("x-instance-id", "-")) c.close() s3_id = sh("docker", "compose", "ps", "-q", "s3").strip() restarts_before = container_events() samplers = [Sampler(cid) for cid in ids] + [Sampler(s3_id)] for s in samplers: s.start() time.sleep(0.5) threads = [threading.Thread(target=upload, args=(i, d, t)) for i, (d, t) in enumerate(files)] wall = time.perf_counter() for t in threads: t.start() for t in threads: t.join() wall = time.perf_counter() - wall time.sleep(0.5) for s in samplers: s.stop() per_replica = {} for cid, s in zip(ids, samplers[:-1]): hostname = sh("docker", "inspect", "-f", "{{.Config.Hostname}}", cid).strip() per_replica[hostname] = { "idle_mib": round(idle[cid] / MiB, 1), "peak_mib": round(cg(cid, "memory.peak") / MiB, 1), "sampled_max_mib": round(max(s.values or [0]) / MiB, 1), "python_vmhwm_mib": round(vmhwm(cid) / MiB, 1), "uploads_served": sum(1 for r in results if r[2] == hostname), "limit_mib": round(cg(cid, "memory.max") / MiB, 1), } per_replica[hostname]["peak_minus_idle_mib"] = round(per_replica[hostname]["peak_mib"] - per_replica[hostname]["idle_mib"], 1) lat = sorted(r[1] for r in results if r[0] == 201) codes: dict[str, int] = {} for r in results: codes[str(r[0])] = codes.get(str(r[0]), 0) + 1 summary = { "scenario": name, "settings": env, "clients": len(files), "s3_sampled_max_mib": round(max(samplers[-1].values or [0]) / MiB, 1), "s3_anon_after_mib": round(memory_stat(s3_id, "anon") / MiB, 1), "s3_limit_mib": round(cg(s3_id, "memory.max") / MiB, 1), # OOM kills / restarts DURING the scenario (any container of the stand): must be empty "incidents": {k: v for k, v in container_events().items() if v != restarts_before.get(k)}, "file_mib_each": round(statistics.mean(len(d) for d, _ in files) / MiB, 2), "types": sorted({t for _, t in files}), "status_codes": codes, "wall_seconds": round(wall, 2), "latency_s": {"p50": round(lat[len(lat) // 2], 2), "max": round(lat[-1], 2)} if lat else None, "replicas": per_replica, } print(json.dumps(summary, ensure_ascii=False), flush=True) return summary def main() -> int: p = argparse.ArgumentParser() p.add_argument("--clients", type=int, default=16) p.add_argument("--output", default="evidence/memory") args = p.parse_args() out = ROOT / args.output out.mkdir(parents=True, exist_ok=True) print("generating files ...", flush=True) pngs = [(imagegen.png_of_size(int(9.5 * MiB)), "image/png") for _ in range(2 * args.clients)] jpegs = [(big_jpeg(8000, 5000, i), "image/jpeg") for i in range(args.clients)] base = {"UPLOAD_CONCURRENCY": "4", "IMAGE_VERIFY_CONCURRENCY": "1"} n, n2 = args.clients, 2 * args.clients scenarios = [ (f"png-9.4MiB x{n}, slots=4 (default)", base, pngs[:n]), (f"png-9.4MiB x{n2}, slots=4 (default)", base, pngs), (f"png-9.4MiB x{n2}, slots=16 (bound raised)", {"UPLOAD_CONCURRENCY": "16", "IMAGE_VERIFY_CONCURRENCY": "1"}, pngs), (f"jpeg-40Mpx-progressive-4:4:4 x{n}, verify=1 (default)", base, jpegs), (f"jpeg-40Mpx-progressive-4:4:4 x{n}, verify=4", {"UPLOAD_CONCURRENCY": "4", "IMAGE_VERIFY_CONCURRENCY": "4"}, jpegs), ] results = [] sh("docker", "compose", "restart", "s3") # S3 memory counters start from an idle process sh("docker", "compose", "up", "-d", "--wait", "s3") try: for name, env, files in scenarios: results.append(run_scenario(name, env, files)) finally: recreate({}) # back to the compose defaults meta = { "date": time.strftime("%Y-%m-%d %H:%M:%S"), "host": platform.platform(), "cpus": os.cpu_count(), "commit": sh("git", "rev-parse", "--short", "HEAD").strip(), "api_limits": sh("docker", "inspect", "-f", "{{.HostConfig.NanoCpus}} {{.HostConfig.Memory}}", replicas()[0]).strip(), "jpeg_file_mib": round(len(jpegs[0][0]) / MiB, 2), "png_file_mib": round(len(pngs[0][0]) / MiB, 2), } (out / "summary.json").write_text(json.dumps({"meta": meta, "scenarios": results}, ensure_ascii=False, indent=1), encoding="utf-8") print(json.dumps(meta, ensure_ascii=False)) return 0 if __name__ == "__main__": sys.exit(main())