#!/usr/bin/env python3 """Experiment: peak memory of API replicas, LB and S3 under N simultaneous ~9.5 MiB uploads. Hypothesis: streaming intake (64 KiB chunks → validator → temp file → S3 PUT from file) keeps the API process RSS growth far below N × file size; total in-flight work per replica is capped by MAX_CONCURRENT_UPLOADS. Method (per concurrency level, fresh replicas): 1. `docker compose restart s3`, then `s3-gw api lb` → new processes; wait for /health/ready; 2. one warm-up upload per replica (imports, pools, TLS-less S3 connections); 3. reset VmHWM of PID 1 (echo 5 > /proc/1/clear_refs) and record baseline; 4. fire N uploads at once through the LB (barrier), sample cgroup memory.stat (anon/file) of every replica, the LB and the s3 container from the host every ~0.25 s; 5. read VmHWM (peak RSS of the uvicorn process) and cgroup memory.peak. Usage: python scripts/upload_memory_experiment.py [--levels 10 20] [--size-mib 9.5] Writes evidence/upload-memory.json. Tokens are kept in memory only. """ from __future__ import annotations import argparse import base64 import json import os import statistics import subprocess import sys import threading import time import urllib.error import urllib.request import uuid from pathlib import Path ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT / "student-tests")) import imagegen # noqa: E402 BASE = os.environ.get("BASE_URL", "http://127.0.0.1:8080").rstrip("/") COMPOSE = ["docker", "compose", "--env-file", ".env"] def sh(args, check=True) -> str: proc = subprocess.run(args, cwd=ROOT, capture_output=True, text=True, encoding="utf-8") if check and proc.returncode != 0: raise RuntimeError(f"{args[:4]} rc={proc.returncode}: {proc.stderr[-300:]}") return proc.stdout def request(method, path, body=None, token=None, raw=None, headers=None, timeout=180): h = {"Accept": "application/json"} if token: h["Authorization"] = "Bearer " + token if body is not None: raw = json.dumps(body).encode() h["Content-Type"] = "application/json" h.update(headers or {}) req = urllib.request.Request(BASE + path, data=raw, headers=h, method=method) try: resp = urllib.request.urlopen(req, timeout=timeout) except urllib.error.HTTPError as err: resp = err data = resp.read() return resp.code, resp.headers, data def actor(label): username = f"mem_{label}_{uuid.uuid4().hex[:10]}" password = "T3st-" + uuid.uuid4().hex status, _, data = request("POST", "/api/v1/users", {"username": username, "display_name": label, "password": password}) assert status == 201, status user = json.loads(data) cred = base64.b64encode(f"{username}:{password}".encode()).decode() status, _, data = request("POST", "/api/v1/auth/sessions", headers={"Authorization": "Basic " + cred}) assert status == 201, status return user["id"], json.loads(data)["token"] def wait_ready(timeout=120): deadline = time.time() + timeout while time.time() < deadline: try: if request("GET", "/health/ready", timeout=3)[0] == 200: seen = {request("GET", "/health/live", timeout=3)[1].get("X-Instance-Id") for _ in range(20)} if len(seen) >= 2: return except OSError: pass time.sleep(1) raise RuntimeError("stack not ready") def containers(): ids = sh(COMPOSE + ["ps", "-q", "api"]).split() return {sh(["docker", "inspect", "-f", "{{.Config.Hostname}}", c]).strip(): c for c in ids} def proc_status(cid): out = sh(["docker", "exec", cid, "cat", "/proc/1/status"]) fields = {} for line in out.splitlines(): if line.startswith(("VmRSS", "VmHWM")): k, v = line.split(":", 1) fields[k] = int(v.split()[0]) // 1024 # MiB return fields def cgroup(cid): stat = sh(["docker", "exec", cid, "cat", "/sys/fs/cgroup/memory.stat"]) vals = dict(line.split() for line in stat.splitlines() if line) peak = sh(["docker", "exec", cid, "cat", "/sys/fs/cgroup/memory.peak"], check=False).strip() current = sh(["docker", "exec", cid, "cat", "/sys/fs/cgroup/memory.current"]).strip() mib = 1024 * 1024 return { "anon_mib": round(int(vals["anon"]) / mib, 1), "file_mib": round(int(vals["file"]) / mib, 1), "current_mib": round(int(current) / mib, 1), "peak_mib": round(int(peak) / mib, 1) if peak.isdigit() else None, } def sampler(cid, stop, out): best = {"anon_mib": 0.0, "file_mib": 0.0, "current_mib": 0.0, "samples": 0} while not stop.is_set(): try: c = cgroup(cid) for k in ("anon_mib", "file_mib", "current_mib"): best[k] = max(best[k], c[k]) best["samples"] += 1 except Exception: pass time.sleep(0.25) out[cid] = best def pct(values, q): values = sorted(values) return round(values[min(len(values) - 1, int(round(q * (len(values) - 1))))], 3) def run_level(n, payload, chat_id, token): print(f"== level {n}: restarting s3, api replicas and lb", flush=True) # s3-gw shares the s3 network namespace and must be restarted after it. sh(COMPOSE + ["restart", "s3"]) sh(COMPOSE + ["restart", "s3-gw", "api", "lb"]) wait_ready() by_host = containers() lb = sh(COMPOSE + ["ps", "-q", "lb"]).split()[0] s3c = sh(COMPOSE + ["ps", "-q", "s3"]).split()[0] # Warm-up: at least one upload served by every replica. served = set() for _ in range(20): _, h, _ = request("POST", f"/api/v1/chats/{chat_id}/attachments?filename=warm.png", token=token, raw=imagegen.rgb_png(64, 64), headers={"Content-Type": "image/png"}) served.add(h.get("X-Instance-Id")) if served >= set(by_host): break baseline = {} for host, cid in by_host.items(): sh(["docker", "exec", cid, "sh", "-c", "echo 5 > /proc/1/clear_refs"]) baseline[host] = {**proc_status(cid), **cgroup(cid)} lb_baseline = cgroup(lb) s3_baseline = cgroup(s3c) stop = threading.Event() sampled = {} threads = [threading.Thread(target=sampler, args=(cid, stop, sampled)) for cid in [*by_host.values(), lb, s3c]] for t in threads: t.start() time.sleep(0.5) barrier = threading.Barrier(n) results = [None] * n def one(i): barrier.wait() t0 = time.monotonic() try: status, h, _ = request("POST", f"/api/v1/chats/{chat_id}/attachments?filename=load{i}.png", token=token, raw=payload, headers={"Content-Type": "image/png"}) results[i] = (status, time.monotonic() - t0, h.get("X-Instance-Id")) except OSError as exc: results[i] = (type(exc).__name__, time.monotonic() - t0, None) workers = [threading.Thread(target=one, args=(i,)) for i in range(n)] started = time.monotonic() for w in workers: w.start() for w in workers: w.join() wall = time.monotonic() - started time.sleep(1.0) stop.set() for t in threads: t.join() replicas = {} for host, cid in by_host.items(): after = proc_status(cid) cg = cgroup(cid) replicas[host] = { "uploads_served": sum(1 for r in results if r[2] == host), "rss_baseline_mib": baseline[host]["VmRSS"], "rss_peak_vmhwm_mib": after["VmHWM"], "rss_peak_growth_mib": after["VmHWM"] - baseline[host]["VmRSS"], "cgroup_peak_mib": cg["peak_mib"], "cgroup_baseline_current_mib": baseline[host]["current_mib"], "sampled_max_anon_mib": sampled.get(cid, {}).get("anon_mib"), "sampled_max_file_cache_mib": sampled.get(cid, {}).get("file_mib"), "samples": sampled.get(cid, {}).get("samples"), } latencies = [r[1] for r in results if r[0] == 201] statuses = {} for r in results: statuses[str(r[0])] = statuses.get(str(r[0]), 0) + 1 per_replica_max = max(v["uploads_served"] for v in replicas.values()) return { "concurrency": n, "payload_bytes": len(payload), "statuses": statuses, "wall_seconds": round(wall, 3), "latency_ok_p50_s": pct(latencies, 0.5) if latencies else None, "latency_ok_p95_s": pct(latencies, 0.95) if latencies else None, "latency_ok_max_s": round(max(latencies), 3) if latencies else None, "throughput_mib_s": round(len(latencies) * len(payload) / (1024 * 1024) / wall, 1), "naive_full_buffer_estimate_mib": round(min(per_replica_max, 8) * len(payload) / (1024 * 1024), 1), "replicas": replicas, "lb": { "mem_limit_mib": int(sh(["docker", "inspect", "-f", "{{.HostConfig.Memory}}", lb]).strip()) // (1024 * 1024), "baseline_current_mib": lb_baseline["current_mib"], "cgroup_peak_mib": cgroup(lb)["peak_mib"], "sampled_max_anon_mib": sampled.get(lb, {}).get("anon_mib"), "sampled_max_current_mib": sampled.get(lb, {}).get("current_mib"), "restarts": int(sh(["docker", "inspect", "-f", "{{.RestartCount}}", lb]).strip()), "oom_killed": sh(["docker", "inspect", "-f", "{{.State.OOMKilled}}", lb]).strip() == "true", }, "s3": _s3_report(s3c, s3_baseline, sampled.get(s3c, {})), } def _s3_report(cid, baseline, sampled): running = sh(["docker", "inspect", "-f", "{{.State.Running}}", cid]).strip() == "true" return { "mem_limit_mib": int(sh(["docker", "inspect", "-f", "{{.HostConfig.Memory}}", cid]).strip()) // (1024 * 1024), "baseline_current_mib": baseline["current_mib"], # memory.peak includes reclaimable page cache of the volume files; anon is the weed heap. "cgroup_peak_mib": cgroup(cid)["peak_mib"] if running else None, "sampled_max_anon_mib": sampled.get("anon_mib"), "sampled_max_current_mib": sampled.get("current_mib"), "running": running, "oom_killed": sh(["docker", "inspect", "-f", "{{.State.OOMKilled}}", cid]).strip() == "true", } def main(): p = argparse.ArgumentParser(description="peak memory under concurrent uploads") p.add_argument("--levels", type=int, nargs="+", default=[10, 20]) p.add_argument("--size-mib", type=float, default=9.5) p.add_argument("--output", default=str(ROOT / "evidence" / "upload-memory.json")) args = p.parse_args() payload = imagegen.large_png(int(args.size_mib * 1024 * 1024)) _, token = actor("a") bob_id, _ = actor("b") me = json.loads(request("GET", "/api/v1/users/me", token=token)[2]) status, _, data = request("POST", "/api/v1/chats", {"title": "memory", "member_ids": [me["id"], bob_id]}, token=token) assert status == 201, status chat_id = json.loads(data)["id"] limits = sh(["docker", "inspect", "-f", "{{.HostConfig.Memory}} {{.HostConfig.NanoCpus}}", list(containers().values())[0]]).split() report = { "date": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "api_mem_limit_mib": int(limits[0]) // (1024 * 1024), "api_cpus": int(limits[1]) / 1e9, "max_concurrent_uploads_per_replica": 8, "levels": [], } for n in args.levels: level = run_level(n, payload, chat_id, token) report["levels"].append(level) print(json.dumps(level, ensure_ascii=False), flush=True) Path(args.output).write_text(json.dumps(report, ensure_ascii=False, indent=1), encoding="utf-8") print("WROTE", args.output) if __name__ == "__main__": main()