Files
mtusi2026lab4/bench/upload_memory.py
T
Alexey BodanyukandClaude Opus 5.5 277d06f5b5 lab4: слот загрузки держится до конца S3 PUT (буфер жив до него); причины 503 в замере
Замер 32 одновременных загрузок показал: семафор ограничивал только чтение тела, а буферы
копились на этапе S3 PUT (+214 МБ при 4 слотах). Теперь пик при 32 клиентах тот же, что при 16.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-25 14:33:56 +03:00

245 lines
11 KiB
Python

#!/usr/bin/env python3
"""Level 4 experiment: peak memory of the API replicas under >= 10 simultaneous uploads.
Hypothesis: the memory of one replica is bounded by UPLOAD_CONCURRENCY x MAX_UPLOAD_BYTES (bodies
waiting for a slot stay in TCP buffers, not in the process) + one image check at a time
(IMAGE_VERIFY_CONCURRENCY), independent of how many clients upload at once.
Method (repeatable, the stand must be up):
* for every scenario the api replicas are RECREATED with the scenario's settings, so the cgroup
counter memory.peak starts from an idle process (it never decreases during a container's life);
* idle memory.current of each replica is read; then N clients upload at the same moment through
the LB (a barrier releases them together), each a different file;
* during the run memory.current of every replica is sampled every 50 ms inside the container;
afterwards memory.peak (cgroup v2: all processes of the container incl. djpeg) and VmHWM of
PID 1 (the Python server alone) are read;
* the normal configuration is restored at the end.
python bench/upload_memory.py --output evidence/memory
"""
from __future__ import annotations
import argparse
import io
import json
import os
import platform
import statistics
import subprocess
import sys
import threading
import time
import uuid
from pathlib import Path
import httpx2 as httpx
from PIL import Image
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "student-tests"))
import imagegen # noqa: E402
BASE = os.environ.get("BASE_URL", "http://127.0.0.1:8080")
PASSWORD = "Bench-password-Ω-12345"
MiB = 1024 * 1024
def sh(*args: str, env: dict | None = None, timeout: float = 300) -> str:
return subprocess.run(args, cwd=ROOT, capture_output=True, text=True, check=True, timeout=timeout,
env={**os.environ, **(env or {})}).stdout
def replicas() -> list[str]:
return sh("docker", "compose", "ps", "-q", "api").split()
def cg(cid: str, name: str) -> int:
return int(sh("docker", "exec", cid, "cat", f"/sys/fs/cgroup/{name}").split()[0])
def memory_stat(cid: str, key: str) -> int:
for line in sh("docker", "exec", cid, "cat", "/sys/fs/cgroup/memory.stat").splitlines():
name, value = line.split()
if name == key:
return int(value)
return 0
def vmhwm(cid: str) -> int:
for line in sh("docker", "exec", cid, "cat", "/proc/1/status").splitlines():
if line.startswith("VmHWM:"):
return int(line.split()[1]) * 1024
return 0
def container_events() -> dict[str, str]:
"""restart count + OOM-kill events per container (cgroup memory.events oom_kill counter)."""
out = {}
for cid in sh("docker", "compose", "ps", "-q").split():
name, restarts = sh("docker", "inspect", "-f", "{{.Name}} {{.RestartCount}}", cid).split()
events = dict(line.split() for line in sh("docker", "exec", cid, "cat", "/sys/fs/cgroup/memory.events").splitlines())
out[name.lstrip("/")] = f"restarts={restarts} oom_kill={events.get('oom_kill', '?')}"
return out
def recreate(env: dict) -> None:
sh("docker", "compose", "up", "-d", "--force-recreate", "--no-deps", "--wait", "api", env=env, timeout=300)
sh(sys.executable, "scripts/lbctl.py", "wait-up", str(len(replicas())), "--timeout", "90") # LB sees them UP
time.sleep(3)
class Sampler(threading.Thread):
"""memory.current of one container every 50 ms, read by a shell loop inside it."""
def __init__(self, cid: str):
super().__init__(daemon=True)
self.cid = cid
self.values: list[int] = []
self.proc = subprocess.Popen(["docker", "exec", cid, "sh", "-c",
"while :; do cat /sys/fs/cgroup/memory.current; sleep 0.05; done"],
stdout=subprocess.PIPE, text=True)
def run(self) -> None:
for line in self.proc.stdout:
if line.strip().isdigit():
self.values.append(int(line))
def stop(self) -> None:
self.proc.kill()
def actor(client: httpx.Client) -> tuple[str, str]:
name = "mem_" + uuid.uuid4().hex[:10]
client.post("/api/v1/users", json={"username": name, "display_name": "bench", "password": PASSWORD}).raise_for_status()
import base64
basic = base64.b64encode(f"{name}:{PASSWORD}".encode()).decode()
token = client.post("/api/v1/auth/sessions", headers={"Authorization": "Basic " + basic}).json()["token"]
me = client.get("/api/v1/users/me", headers={"Authorization": "Bearer " + token}).json()
chat = client.post("/api/v1/chats", headers={"Authorization": "Bearer " + token},
json={"title": "mem", "member_ids": [me["id"]]}).json()
return token, chat["id"]
def big_jpeg(width: int, height: int, seed: int) -> bytes:
"""Large progressive JPEG (smooth content keeps it < 10 MiB): the costliest file to check."""
image = Image.linear_gradient("L").resize((width, height)).convert("RGB")
image.putpixel((seed % width, 0), (seed % 255, 0, 0))
out = io.BytesIO()
image.save(out, "JPEG", quality=90, progressive=True, subsampling=0)
return out.getvalue()
def run_scenario(name: str, env: dict, files: list[tuple[bytes, str]]) -> dict:
recreate(env)
ids = replicas()
idle = {cid: cg(cid, "memory.current") for cid in ids}
client = httpx.Client(base_url=BASE, timeout=120)
token, chat_id = actor(client)
barrier = threading.Barrier(len(files))
results: list[tuple[int, float, str]] = [None] * len(files) # type: ignore[list-item]
def upload(i: int, data: bytes, ctype: str) -> None:
c = httpx.Client(base_url=BASE, timeout=120)
barrier.wait()
started = time.perf_counter()
r = c.post(f"/api/v1/chats/{chat_id}/attachments?filename=f{i}", content=data,
headers={"Authorization": "Bearer " + token, "Content-Type": ctype})
reason = r.json()["error"]["message"] if r.status_code != 201 else ""
results[i] = (r.status_code, time.perf_counter() - started, r.headers.get("x-instance-id", "-"), reason)
c.close()
s3_id = sh("docker", "compose", "ps", "-q", "s3").strip()
restarts_before = container_events()
samplers = [Sampler(cid) for cid in ids] + [Sampler(s3_id)]
for s in samplers:
s.start()
time.sleep(0.5)
threads = [threading.Thread(target=upload, args=(i, d, t)) for i, (d, t) in enumerate(files)]
wall = time.perf_counter()
for t in threads:
t.start()
for t in threads:
t.join()
wall = time.perf_counter() - wall
time.sleep(0.5)
for s in samplers:
s.stop()
per_replica = {}
for cid, s in zip(ids, samplers[:-1]):
hostname = sh("docker", "inspect", "-f", "{{.Config.Hostname}}", cid).strip()
per_replica[hostname] = {
"idle_mib": round(idle[cid] / MiB, 1),
"peak_mib": round(cg(cid, "memory.peak") / MiB, 1),
"sampled_max_mib": round(max(s.values or [0]) / MiB, 1),
"python_vmhwm_mib": round(vmhwm(cid) / MiB, 1),
"uploads_served": sum(1 for r in results if r[2] == hostname),
"limit_mib": round(cg(cid, "memory.max") / MiB, 1),
}
per_replica[hostname]["peak_minus_idle_mib"] = round(per_replica[hostname]["peak_mib"]
- per_replica[hostname]["idle_mib"], 1)
lat = sorted(r[1] for r in results if r[0] == 201)
codes: dict[str, int] = {}
reasons: dict[str, int] = {}
for r in results:
codes[str(r[0])] = codes.get(str(r[0]), 0) + 1
if r[3]:
reasons[r[3]] = reasons.get(r[3], 0) + 1
summary = {
"scenario": name, "settings": env, "clients": len(files),
"s3_sampled_max_mib": round(max(samplers[-1].values or [0]) / MiB, 1),
"s3_anon_after_mib": round(memory_stat(s3_id, "anon") / MiB, 1),
"s3_limit_mib": round(cg(s3_id, "memory.max") / MiB, 1),
# OOM kills / restarts DURING the scenario (any container of the stand): must be empty
"incidents": {k: v for k, v in container_events().items() if v != restarts_before.get(k)},
"file_mib_each": round(statistics.mean(len(d) for d, _ in files) / MiB, 2),
"types": sorted({t for _, t in files}), "status_codes": codes, "error_reasons": reasons, "wall_seconds": round(wall, 2),
"latency_s": {"p50": round(lat[len(lat) // 2], 2), "max": round(lat[-1], 2)} if lat else None,
"replicas": per_replica,
}
print(json.dumps(summary, ensure_ascii=False), flush=True)
return summary
def main() -> int:
p = argparse.ArgumentParser()
p.add_argument("--clients", type=int, default=16)
p.add_argument("--output", default="evidence/memory")
args = p.parse_args()
out = ROOT / args.output
out.mkdir(parents=True, exist_ok=True)
print("generating files ...", flush=True)
pngs = [(imagegen.png_of_size(int(9.5 * MiB)), "image/png") for _ in range(2 * args.clients)]
jpegs = [(big_jpeg(8000, 5000, i), "image/jpeg") for i in range(args.clients)]
base = {"UPLOAD_CONCURRENCY": "4", "IMAGE_VERIFY_CONCURRENCY": "1"}
n, n2 = args.clients, 2 * args.clients
scenarios = [
(f"png-9.4MiB x{n}, slots=4 (default)", base, pngs[:n]),
(f"png-9.4MiB x{n2}, slots=4 (default)", base, pngs),
(f"png-9.4MiB x{n2}, slots=16 (bound raised)", {"UPLOAD_CONCURRENCY": "16", "IMAGE_VERIFY_CONCURRENCY": "1"}, pngs),
(f"jpeg-40Mpx-progressive-4:4:4 x{n}, verify=1 (default)", base, jpegs),
(f"jpeg-40Mpx-progressive-4:4:4 x{n}, verify=4", {"UPLOAD_CONCURRENCY": "4", "IMAGE_VERIFY_CONCURRENCY": "4"}, jpegs),
]
results = []
sh("docker", "compose", "restart", "s3") # S3 memory counters start from an idle process
sh("docker", "compose", "up", "-d", "--wait", "s3")
try:
for name, env, files in scenarios:
results.append(run_scenario(name, env, files))
finally:
recreate({}) # back to the compose defaults
meta = {
"date": time.strftime("%Y-%m-%d %H:%M:%S"), "host": platform.platform(), "cpus": os.cpu_count(),
"commit": sh("git", "rev-parse", "--short", "HEAD").strip(),
"api_limits": sh("docker", "inspect", "-f", "{{.HostConfig.NanoCpus}} {{.HostConfig.Memory}}",
replicas()[0]).strip(),
"jpeg_file_mib": round(len(jpegs[0][0]) / MiB, 2), "png_file_mib": round(len(pngs[0][0]) / MiB, 2),
}
(out / "summary.json").write_text(json.dumps({"meta": meta, "scenarios": results}, ensure_ascii=False, indent=1),
encoding="utf-8")
print(json.dumps(meta, ensure_ascii=False))
return 0
if __name__ == "__main__":
sys.exit(main())