Замер 32 одновременных загрузок показал: семафор ограничивал только чтение тела, а буферы копились на этапе S3 PUT (+214 МБ при 4 слотах). Теперь пик при 32 клиентах тот же, что при 16. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
245 lines
11 KiB
Python
245 lines
11 KiB
Python
#!/usr/bin/env python3
|
|
"""Level 4 experiment: peak memory of the API replicas under >= 10 simultaneous uploads.
|
|
|
|
Hypothesis: the memory of one replica is bounded by UPLOAD_CONCURRENCY x MAX_UPLOAD_BYTES (bodies
|
|
waiting for a slot stay in TCP buffers, not in the process) + one image check at a time
|
|
(IMAGE_VERIFY_CONCURRENCY), independent of how many clients upload at once.
|
|
|
|
Method (repeatable, the stand must be up):
|
|
* for every scenario the api replicas are RECREATED with the scenario's settings, so the cgroup
|
|
counter memory.peak starts from an idle process (it never decreases during a container's life);
|
|
* idle memory.current of each replica is read; then N clients upload at the same moment through
|
|
the LB (a barrier releases them together), each a different file;
|
|
* during the run memory.current of every replica is sampled every 50 ms inside the container;
|
|
afterwards memory.peak (cgroup v2: all processes of the container incl. djpeg) and VmHWM of
|
|
PID 1 (the Python server alone) are read;
|
|
* the normal configuration is restored at the end.
|
|
|
|
python bench/upload_memory.py --output evidence/memory
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import io
|
|
import json
|
|
import os
|
|
import platform
|
|
import statistics
|
|
import subprocess
|
|
import sys
|
|
import threading
|
|
import time
|
|
import uuid
|
|
from pathlib import Path
|
|
|
|
import httpx2 as httpx
|
|
from PIL import Image
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT / "student-tests"))
|
|
import imagegen # noqa: E402
|
|
|
|
BASE = os.environ.get("BASE_URL", "http://127.0.0.1:8080")
|
|
PASSWORD = "Bench-password-Ω-12345"
|
|
MiB = 1024 * 1024
|
|
|
|
|
|
def sh(*args: str, env: dict | None = None, timeout: float = 300) -> str:
|
|
return subprocess.run(args, cwd=ROOT, capture_output=True, text=True, check=True, timeout=timeout,
|
|
env={**os.environ, **(env or {})}).stdout
|
|
|
|
|
|
def replicas() -> list[str]:
|
|
return sh("docker", "compose", "ps", "-q", "api").split()
|
|
|
|
|
|
def cg(cid: str, name: str) -> int:
|
|
return int(sh("docker", "exec", cid, "cat", f"/sys/fs/cgroup/{name}").split()[0])
|
|
|
|
|
|
def memory_stat(cid: str, key: str) -> int:
|
|
for line in sh("docker", "exec", cid, "cat", "/sys/fs/cgroup/memory.stat").splitlines():
|
|
name, value = line.split()
|
|
if name == key:
|
|
return int(value)
|
|
return 0
|
|
|
|
|
|
def vmhwm(cid: str) -> int:
|
|
for line in sh("docker", "exec", cid, "cat", "/proc/1/status").splitlines():
|
|
if line.startswith("VmHWM:"):
|
|
return int(line.split()[1]) * 1024
|
|
return 0
|
|
|
|
|
|
def container_events() -> dict[str, str]:
|
|
"""restart count + OOM-kill events per container (cgroup memory.events oom_kill counter)."""
|
|
out = {}
|
|
for cid in sh("docker", "compose", "ps", "-q").split():
|
|
name, restarts = sh("docker", "inspect", "-f", "{{.Name}} {{.RestartCount}}", cid).split()
|
|
events = dict(line.split() for line in sh("docker", "exec", cid, "cat", "/sys/fs/cgroup/memory.events").splitlines())
|
|
out[name.lstrip("/")] = f"restarts={restarts} oom_kill={events.get('oom_kill', '?')}"
|
|
return out
|
|
|
|
|
|
def recreate(env: dict) -> None:
|
|
sh("docker", "compose", "up", "-d", "--force-recreate", "--no-deps", "--wait", "api", env=env, timeout=300)
|
|
sh(sys.executable, "scripts/lbctl.py", "wait-up", str(len(replicas())), "--timeout", "90") # LB sees them UP
|
|
time.sleep(3)
|
|
|
|
|
|
class Sampler(threading.Thread):
|
|
"""memory.current of one container every 50 ms, read by a shell loop inside it."""
|
|
|
|
def __init__(self, cid: str):
|
|
super().__init__(daemon=True)
|
|
self.cid = cid
|
|
self.values: list[int] = []
|
|
self.proc = subprocess.Popen(["docker", "exec", cid, "sh", "-c",
|
|
"while :; do cat /sys/fs/cgroup/memory.current; sleep 0.05; done"],
|
|
stdout=subprocess.PIPE, text=True)
|
|
|
|
def run(self) -> None:
|
|
for line in self.proc.stdout:
|
|
if line.strip().isdigit():
|
|
self.values.append(int(line))
|
|
|
|
def stop(self) -> None:
|
|
self.proc.kill()
|
|
|
|
|
|
def actor(client: httpx.Client) -> tuple[str, str]:
|
|
name = "mem_" + uuid.uuid4().hex[:10]
|
|
client.post("/api/v1/users", json={"username": name, "display_name": "bench", "password": PASSWORD}).raise_for_status()
|
|
import base64
|
|
basic = base64.b64encode(f"{name}:{PASSWORD}".encode()).decode()
|
|
token = client.post("/api/v1/auth/sessions", headers={"Authorization": "Basic " + basic}).json()["token"]
|
|
me = client.get("/api/v1/users/me", headers={"Authorization": "Bearer " + token}).json()
|
|
chat = client.post("/api/v1/chats", headers={"Authorization": "Bearer " + token},
|
|
json={"title": "mem", "member_ids": [me["id"]]}).json()
|
|
return token, chat["id"]
|
|
|
|
|
|
def big_jpeg(width: int, height: int, seed: int) -> bytes:
|
|
"""Large progressive JPEG (smooth content keeps it < 10 MiB): the costliest file to check."""
|
|
image = Image.linear_gradient("L").resize((width, height)).convert("RGB")
|
|
image.putpixel((seed % width, 0), (seed % 255, 0, 0))
|
|
out = io.BytesIO()
|
|
image.save(out, "JPEG", quality=90, progressive=True, subsampling=0)
|
|
return out.getvalue()
|
|
|
|
|
|
def run_scenario(name: str, env: dict, files: list[tuple[bytes, str]]) -> dict:
|
|
recreate(env)
|
|
ids = replicas()
|
|
idle = {cid: cg(cid, "memory.current") for cid in ids}
|
|
client = httpx.Client(base_url=BASE, timeout=120)
|
|
token, chat_id = actor(client)
|
|
barrier = threading.Barrier(len(files))
|
|
results: list[tuple[int, float, str]] = [None] * len(files) # type: ignore[list-item]
|
|
|
|
def upload(i: int, data: bytes, ctype: str) -> None:
|
|
c = httpx.Client(base_url=BASE, timeout=120)
|
|
barrier.wait()
|
|
started = time.perf_counter()
|
|
r = c.post(f"/api/v1/chats/{chat_id}/attachments?filename=f{i}", content=data,
|
|
headers={"Authorization": "Bearer " + token, "Content-Type": ctype})
|
|
reason = r.json()["error"]["message"] if r.status_code != 201 else ""
|
|
results[i] = (r.status_code, time.perf_counter() - started, r.headers.get("x-instance-id", "-"), reason)
|
|
c.close()
|
|
|
|
s3_id = sh("docker", "compose", "ps", "-q", "s3").strip()
|
|
restarts_before = container_events()
|
|
samplers = [Sampler(cid) for cid in ids] + [Sampler(s3_id)]
|
|
for s in samplers:
|
|
s.start()
|
|
time.sleep(0.5)
|
|
threads = [threading.Thread(target=upload, args=(i, d, t)) for i, (d, t) in enumerate(files)]
|
|
wall = time.perf_counter()
|
|
for t in threads:
|
|
t.start()
|
|
for t in threads:
|
|
t.join()
|
|
wall = time.perf_counter() - wall
|
|
time.sleep(0.5)
|
|
for s in samplers:
|
|
s.stop()
|
|
per_replica = {}
|
|
for cid, s in zip(ids, samplers[:-1]):
|
|
hostname = sh("docker", "inspect", "-f", "{{.Config.Hostname}}", cid).strip()
|
|
per_replica[hostname] = {
|
|
"idle_mib": round(idle[cid] / MiB, 1),
|
|
"peak_mib": round(cg(cid, "memory.peak") / MiB, 1),
|
|
"sampled_max_mib": round(max(s.values or [0]) / MiB, 1),
|
|
"python_vmhwm_mib": round(vmhwm(cid) / MiB, 1),
|
|
"uploads_served": sum(1 for r in results if r[2] == hostname),
|
|
"limit_mib": round(cg(cid, "memory.max") / MiB, 1),
|
|
}
|
|
per_replica[hostname]["peak_minus_idle_mib"] = round(per_replica[hostname]["peak_mib"]
|
|
- per_replica[hostname]["idle_mib"], 1)
|
|
lat = sorted(r[1] for r in results if r[0] == 201)
|
|
codes: dict[str, int] = {}
|
|
reasons: dict[str, int] = {}
|
|
for r in results:
|
|
codes[str(r[0])] = codes.get(str(r[0]), 0) + 1
|
|
if r[3]:
|
|
reasons[r[3]] = reasons.get(r[3], 0) + 1
|
|
summary = {
|
|
"scenario": name, "settings": env, "clients": len(files),
|
|
"s3_sampled_max_mib": round(max(samplers[-1].values or [0]) / MiB, 1),
|
|
"s3_anon_after_mib": round(memory_stat(s3_id, "anon") / MiB, 1),
|
|
"s3_limit_mib": round(cg(s3_id, "memory.max") / MiB, 1),
|
|
# OOM kills / restarts DURING the scenario (any container of the stand): must be empty
|
|
"incidents": {k: v for k, v in container_events().items() if v != restarts_before.get(k)},
|
|
"file_mib_each": round(statistics.mean(len(d) for d, _ in files) / MiB, 2),
|
|
"types": sorted({t for _, t in files}), "status_codes": codes, "error_reasons": reasons, "wall_seconds": round(wall, 2),
|
|
"latency_s": {"p50": round(lat[len(lat) // 2], 2), "max": round(lat[-1], 2)} if lat else None,
|
|
"replicas": per_replica,
|
|
}
|
|
print(json.dumps(summary, ensure_ascii=False), flush=True)
|
|
return summary
|
|
|
|
|
|
def main() -> int:
|
|
p = argparse.ArgumentParser()
|
|
p.add_argument("--clients", type=int, default=16)
|
|
p.add_argument("--output", default="evidence/memory")
|
|
args = p.parse_args()
|
|
out = ROOT / args.output
|
|
out.mkdir(parents=True, exist_ok=True)
|
|
print("generating files ...", flush=True)
|
|
pngs = [(imagegen.png_of_size(int(9.5 * MiB)), "image/png") for _ in range(2 * args.clients)]
|
|
jpegs = [(big_jpeg(8000, 5000, i), "image/jpeg") for i in range(args.clients)]
|
|
base = {"UPLOAD_CONCURRENCY": "4", "IMAGE_VERIFY_CONCURRENCY": "1"}
|
|
n, n2 = args.clients, 2 * args.clients
|
|
scenarios = [
|
|
(f"png-9.4MiB x{n}, slots=4 (default)", base, pngs[:n]),
|
|
(f"png-9.4MiB x{n2}, slots=4 (default)", base, pngs),
|
|
(f"png-9.4MiB x{n2}, slots=16 (bound raised)", {"UPLOAD_CONCURRENCY": "16", "IMAGE_VERIFY_CONCURRENCY": "1"}, pngs),
|
|
(f"jpeg-40Mpx-progressive-4:4:4 x{n}, verify=1 (default)", base, jpegs),
|
|
(f"jpeg-40Mpx-progressive-4:4:4 x{n}, verify=4", {"UPLOAD_CONCURRENCY": "4", "IMAGE_VERIFY_CONCURRENCY": "4"}, jpegs),
|
|
]
|
|
results = []
|
|
sh("docker", "compose", "restart", "s3") # S3 memory counters start from an idle process
|
|
sh("docker", "compose", "up", "-d", "--wait", "s3")
|
|
try:
|
|
for name, env, files in scenarios:
|
|
results.append(run_scenario(name, env, files))
|
|
finally:
|
|
recreate({}) # back to the compose defaults
|
|
meta = {
|
|
"date": time.strftime("%Y-%m-%d %H:%M:%S"), "host": platform.platform(), "cpus": os.cpu_count(),
|
|
"commit": sh("git", "rev-parse", "--short", "HEAD").strip(),
|
|
"api_limits": sh("docker", "inspect", "-f", "{{.HostConfig.NanoCpus}} {{.HostConfig.Memory}}",
|
|
replicas()[0]).strip(),
|
|
"jpeg_file_mib": round(len(jpegs[0][0]) / MiB, 2), "png_file_mib": round(len(pngs[0][0]) / MiB, 2),
|
|
}
|
|
(out / "summary.json").write_text(json.dumps({"meta": meta, "scenarios": results}, ensure_ascii=False, indent=1),
|
|
encoding="utf-8")
|
|
print(json.dumps(meta, ensure_ascii=False))
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|