Files
mtusi2026lab4/scripts/lab4/attachments_bench.py
T

294 lines
11 KiB
Python
Executable File

#!/usr/bin/env python3
"""Upload benchmark: API traffic and peak RSS per upload for proxy vs direct mode.
attachments_bench.py run --mode proxy --size-bytes 1048576 --concurrency 10 \
--uploads-per-worker 5 --repeat 3 --output evidence/lab4/bench/proxy-1048576-c10.json
attachments_bench.py summarize evidence/lab4/bench/*.json > evidence/lab4/summary.md
Run from the repository root with the stand up. Tokens, signed URLs and secrets
are never written to the output.
"""
import argparse
import base64
import hashlib
import http.client
import json
import os
import statistics
import subprocess
import sys
import threading
import time
import urllib.parse
import uuid
from datetime import datetime, timezone
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from pngkit import noise_png # noqa: E402
def sh(*args, check=True):
return subprocess.run(args, check=check, capture_output=True, text=True).stdout.strip()
def read_env(path=".env"):
values = {}
with open(path) as handle:
for line in handle:
line = line.strip()
if line and not line.startswith("#") and "=" in line:
key, _, value = line.partition("=")
values[key] = value
return values
def request(host, port, method, path, headers=None, body=None):
connection = http.client.HTTPConnection(host, port, timeout=120)
try:
connection.request(method, path, body=body, headers=headers or {})
response = connection.getresponse()
payload = response.read()
return response.status, payload
finally:
connection.close()
def api_json(env, method, path, token=None, body=None, basic=None):
headers = {}
data = None
if body is not None:
headers["content-type"] = "application/json"
data = json.dumps(body).encode()
if token:
headers["authorization"] = f"Bearer {token}"
if basic:
headers["authorization"] = "Basic " + base64.b64encode(basic.encode()).decode()
port = int(env.get("APP_PUBLIC_PORT", "8080"))
status, payload = request("127.0.0.1", port, method, path, headers, data)
return status, json.loads(payload) if payload else None
def containers():
ids = sh("docker", "compose", "--env-file", ".env", "ps", "-q", "api").split()
return sorted(ids)
NODE_METRICS = (
"fetch('http://127.0.0.1:'+(process.env.APP_PORT||8080)+'/internal/metrics'+process.argv[1],"
"{method:process.argv[1]?'POST':'GET',headers:{authorization:'Bearer '+process.env.METRICS_TOKEN}})"
".then(r=>r.text()).then(console.log)"
)
def metrics(container):
return json.loads(sh("docker", "exec", container, "node", "-e", NODE_METRICS, ""))
def reset_metrics(container):
sh("docker", "exec", container, "node", "-e", NODE_METRICS, "/reset")
def net_bytes(container):
for line in sh("docker", "exec", container, "cat", "/proc/net/dev").splitlines():
if line.strip().startswith("eth0:"):
fields = line.split(":", 1)[1].split()
return int(fields[0]), int(fields[8])
raise RuntimeError("eth0 not found")
def percentile(values, fraction):
ordered = sorted(values)
if not ordered:
return 0.0
return ordered[min(len(ordered) - 1, max(0, int(round(fraction * len(ordered) + 0.5)) - 1))]
def one_upload(env, mode, token, chat_id, data, seed):
digest = hashlib.sha256(data).hexdigest()
filename = f"bench-{seed}.png"
port = int(env.get("APP_PUBLIC_PORT", "8080"))
started = time.perf_counter()
if mode == "proxy":
status, payload = request(
"127.0.0.1", port, "POST",
f"/api/v1/chats/{chat_id}/attachments?filename={urllib.parse.quote(filename)}",
{"authorization": f"Bearer {token}", "content-type": "image/png"}, data,
)
ok = status == 201
attachment = json.loads(payload) if ok else None
else:
status, ticket = api_json(
env, "POST", f"/api/v1/chats/{chat_id}/attachments/direct-uploads", token,
{"filename": filename, "content_type": "image/png", "size_bytes": len(data), "sha256": digest},
)
ok = status == 201
attachment = None
if ok:
target = urllib.parse.urlsplit(ticket["url"])
put_status, _ = request(
target.hostname, target.port, "PUT", target.path + "?" + target.query,
{**ticket["headers"]}, data,
)
ok = put_status == 200
if ok:
status, attachment = api_json(
env, "POST", f"/api/v1/attachments/{ticket['attachment_id']}/complete", token
)
ok = status == 200
elapsed_ms = (time.perf_counter() - started) * 1000
if ok and (attachment is None or attachment["sha256"] != digest):
ok = False
return ok, elapsed_ms
def run_once(env, args, token, chat_id, ids):
for container in ids:
reset_metrics(container)
time.sleep(0.2)
baseline = {c: metrics(c) for c in ids}
net_before = {c: net_bytes(c) for c in ids}
latencies, failures = [], []
lock = threading.Lock()
def worker(index):
for step in range(args.uploads_per_worker):
seed = index * 1000 + step
data = noise_png(args.size_bytes, seed)
ok, elapsed = one_upload(env, args.mode, token, chat_id, data, seed)
with lock:
(latencies if ok else failures).append(elapsed)
started = time.perf_counter()
threads = [threading.Thread(target=worker, args=(i,)) for i in range(args.concurrency)]
for thread in threads:
thread.start()
for thread in threads:
thread.join()
duration = time.perf_counter() - started
time.sleep(0.2)
after = {c: metrics(c) for c in ids}
net_after = {c: net_bytes(c) for c in ids}
replicas = []
for c in ids:
base_rss = baseline[c]["memory_bytes"]["rss"]
peak = after[c]["memory_peak_bytes"]
replicas.append({
"instance_id": after[c]["instance_id"],
"delta_rx_bytes": net_after[c][0] - net_before[c][0],
"delta_tx_bytes": net_after[c][1] - net_before[c][1],
"baseline_rss_bytes": base_rss,
"peak_memory_bytes": peak,
"delta_rss_bytes": peak["rss"] - base_rss,
"counters": {k: v for k, v in after[c]["counters"].items() if k.startswith("attachments.")},
})
total = args.concurrency * args.uploads_per_worker
return {
"uploads": total,
"succeeded": len(latencies),
"failed": len(failures),
"duration_s": round(duration, 3),
"latency_ms": {
"p50": round(percentile(latencies, 0.50), 1),
"p95": round(percentile(latencies, 0.95), 1),
"p99": round(percentile(latencies, 0.99), 1),
},
"replicas": replicas,
"total_delta_rx_bytes": sum(r["delta_rx_bytes"] for r in replicas),
"total_delta_tx_bytes": sum(r["delta_tx_bytes"] for r in replicas),
"rx_bytes_per_upload": round(sum(r["delta_rx_bytes"] for r in replicas) / total),
"tx_bytes_per_upload": round(sum(r["delta_tx_bytes"] for r in replicas) / total),
"max_delta_rss_bytes": max(r["delta_rss_bytes"] for r in replicas),
}
def command_run(args):
env = read_env()
ids = containers()
if not ids:
sys.exit("no api containers found")
name = f"bench_{uuid.uuid4().hex[:10]}"
password = "bench:" + uuid.uuid4().hex
status, user = api_json(env, "POST", "/api/v1/users",
body={"username": name, "display_name": "bench", "password": password})
assert status == 201, status
status, session = api_json(env, "POST", "/api/v1/auth/sessions", basic=f"{name}:{password}")
assert status == 201, status
token = session["token"]
status, chat = api_json(env, "POST", "/api/v1/chats", token, {"title": "bench", "member_ids": [user["id"]]})
assert status == 201, status
# Warm-up: one upload per replica path, not measured.
for seed in range(len(ids) * 2):
one_upload(env, args.mode, token, chat["id"], noise_png(args.size_bytes, 900000 + seed), seed)
limits = []
for c in ids:
limits.append(sh("docker", "inspect", "-f", "{{.HostConfig.NanoCpus}} {{.HostConfig.Memory}}", c))
repeats = [run_once(env, args, token, chat["id"], ids) for _ in range(args.repeat)]
mem_total = next(int(l.split()[1]) * 1024 for l in open("/proc/meminfo") if l.startswith("MemTotal"))
result = {
"date": datetime.now(timezone.utc).isoformat(timespec="seconds"),
"git": sh("git", "rev-parse", "--short", "HEAD", check=False),
"params": {
"mode": args.mode, "size_bytes": args.size_bytes, "concurrency": args.concurrency,
"uploads_per_worker": args.uploads_per_worker, "repeat": args.repeat,
},
"host": {"nproc": os.cpu_count(), "mem_total_bytes": mem_total},
"api_container_limits_nanocpus_memory": limits,
"repeats": repeats,
}
os.makedirs(os.path.dirname(os.path.abspath(args.output)), exist_ok=True)
with open(args.output, "w") as handle:
json.dump(result, handle, indent=2)
handle.write("\n")
for index, item in enumerate(repeats):
print(f"repeat {index + 1}: ok={item['succeeded']}/{item['uploads']} "
f"rx/upload={item['rx_bytes_per_upload']} tx/upload={item['tx_bytes_per_upload']} "
f"maxΔrss={item['max_delta_rss_bytes']} p50={item['latency_ms']['p50']}ms")
def mib(value):
return f"{value / 1048576:.2f}"
def command_summarize(args):
rows = []
for path in args.paths:
with open(path) as handle:
data = json.load(handle)
p = data["params"]
reps = data["repeats"]
med = lambda key: statistics.median(r[key] for r in reps) # noqa: E731
rows.append((p["mode"], p["size_bytes"], p["concurrency"], reps, med, data))
rows.sort(key=lambda r: (r[1], r[0], r[2]))
print("| mode | size, MiB | C | ok | API rx/upload, MiB | API tx/upload, MiB | max Δrss, MiB | p50, ms | p95, ms |")
print("|---|---|---|---|---|---|---|---|---|")
for mode, size, conc, reps, med, _data in rows:
ok = sum(r["succeeded"] for r in reps)
total = sum(r["uploads"] for r in reps)
p50 = statistics.median(r["latency_ms"]["p50"] for r in reps)
p95 = statistics.median(r["latency_ms"]["p95"] for r in reps)
print(f"| {mode} | {mib(size)} | {conc} | {ok}/{total} | {mib(med('rx_bytes_per_upload'))} | "
f"{mib(med('tx_bytes_per_upload'))} | {mib(med('max_delta_rss_bytes'))} | {p50:.0f} | {p95:.0f} |")
def main():
parser = argparse.ArgumentParser()
sub = parser.add_subparsers(dest="command", required=True)
run = sub.add_parser("run")
run.add_argument("--mode", choices=["proxy", "direct"], required=True)
run.add_argument("--size-bytes", type=int, required=True)
run.add_argument("--concurrency", type=int, default=10)
run.add_argument("--uploads-per-worker", type=int, default=5)
run.add_argument("--repeat", type=int, default=3)
run.add_argument("--output", required=True)
summarize = sub.add_parser("summarize")
summarize.add_argument("paths", nargs="+")
args = parser.parse_args()
{"run": command_run, "summarize": command_summarize}[args.command](args)
if __name__ == "__main__":
main()