Files
mtusi2026labs/scripts/upload_memory.py
T
ArtyomandClaude Opus 5.5 5ae3d77152 Carry over the lab 5 solution
Bring the application, own tests, Dockerfile, toolchain, compose stack,
HAProxy, k6, RustFS and RabbitMQ configuration, migrations, hooks,
scenario scripts and the CI job from the lab 5 solution commit 64d5c67,
keep the lab 5 Makefile targets on top of the unchanged skeleton
targets, and append the lab 5 solution variables to the skeleton
.env.example. The package and the application follow the contract
version 1.6.0, and the image, the throwaway test containers and the
broker vhost description are tagged lab6 so they do not collide with a
lab 5 stand on the same host.

This is the starting point of the lab 6 cycle: lint, own tests and the
offline skeleton check pass before any lab 6 change.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-10-29 03:51:58 +03:00

118 lines
4.1 KiB
Python
Executable File

#!/usr/bin/env python3
import argparse
import json
import os
import subprocess
import sys
import time
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "scripts"))
seed = __import__("lab4_seed")
FORMAT = "{:>5} {:>5} {:>4} {:<18} {:>7} {:>7} {:>8} {:>8} {:>7} {:>8} {:>6}"
COLUMNS = FORMAT.format(
"slots",
"size",
"conc",
"codes",
"p50_ms",
"p95_ms",
"api_idle",
"api_peak",
"api_oom",
"lb_peak",
"lb_oom",
)
OVERRIDES = ("UPLOAD_CONCURRENCY", "MALLOC_MMAP_THRESHOLD_")
def recreate(slots):
env = dict(os.environ, UPLOAD_CONCURRENCY=str(slots))
if not slots:
env = {k: v for k, v in os.environ.items() if k not in OVERRIDES}
cmd = ["docker", "compose", "up", "-d", "--no-deps", "--force-recreate", "--wait", "api", "lb"]
subprocess.run(cmd, cwd=ROOT, env=env, check=True, capture_output=True)
subprocess.run([str(ROOT / "scripts/lb-wait.sh")], cwd=ROOT, check=True)
def balancer():
cid = seed.run(["docker", "compose", "ps", "-q", "lb"]).strip()
events = dict(row.split() for row in seed.cgroup(cid, "memory.events").splitlines())
peak = seed.cgroup(cid, "memory.peak")
return {
"cgroup_memory_peak_mib": round(int(peak) / 1048576, 1),
"oom_kill": int(events["oom_kill"]),
}
def line(report):
memory = report["memory"].values()
codes = ",".join(f"{k}x{v}" for k, v in sorted(report["status_codes"].items()))
return FORMAT.format(
report["upload_concurrency"],
f"{report['body_bytes'] / 1048576:.1f}M",
report["concurrency"],
codes,
report["latency_ms"]["p50"],
report["latency_ms"]["p95"],
max(m["idle_mib"] for m in memory),
max(m["cgroup_memory_peak_mib"] or 0 for m in memory),
sum(m["oom_kill"] for m in memory),
report["lb"]["cgroup_memory_peak_mib"],
report["lb"]["oom_kill"],
)
def main():
p = argparse.ArgumentParser(description="Peak memory of API replicas under concurrent uploads")
p.add_argument("--base-url", default=os.environ.get("BASE_URL", "http://localhost:8080"))
p.add_argument("--slots", type=int, nargs="+", default=[64, 3])
p.add_argument("--sizes", type=int, nargs="+", default=[1 << 20, 10 << 20])
p.add_argument("--levels", type=int, nargs="+", default=[10, 20, 40])
p.add_argument("--rounds", type=int, default=2, help="uploads per concurrent client")
p.add_argument("--out", default="evidence/upload-memory")
args = p.parse_args()
out = ROOT / args.out
out.mkdir(parents=True, exist_ok=True)
commit = seed.run(["git", "rev-parse", "--short", "HEAD"]).strip()
threshold = os.environ.get("MALLOC_MMAP_THRESHOLD_", "131072")
lines = [
f"# {time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())} commit {commit}; api x2: 1 CPU,"
f" 512 MiB, 2 workers; MALLOC_MMAP_THRESHOLD_={threshold}; {args.rounds} uploads"
" per client; peaks from cgroup memory.peak, MiB",
COLUMNS,
]
print("\n".join(lines), flush=True)
try:
for slots in args.slots:
for size in args.sizes:
for level in args.levels:
recreate(slots)
run = argparse.Namespace(
base_url=args.base_url,
timeout=60,
size=size,
count=level * args.rounds,
concurrency=level,
sample=True,
service="api",
link=0,
ids_out=None,
)
report = {"upload_concurrency": slots, **seed.over_http(run)}
report["lb"] = balancer()
name = f"slots{slots}-{size >> 20}mib-c{level}.json"
(out / name).write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n")
lines.append(line(report))
print(lines[-1], flush=True)
finally:
recreate(None)
(out / "summary.txt").write_text("\n".join(lines) + "\n")
if __name__ == "__main__":
main()