341 lines
14 KiB
Python
Executable File
341 lines
14 KiB
Python
Executable File
#!/usr/bin/env python3
|
||
"""Upload benchmark: API traffic and peak RSS per upload for proxy vs direct mode.
|
||
|
||
attachments_bench.py run --mode proxy --size-bytes 1048576 --concurrency 10 \
|
||
--uploads-per-worker 5 --repeat 3 --output evidence/lab4/bench/proxy-1048576-c10.json
|
||
attachments_bench.py summarize evidence/lab4/bench/*.json > evidence/lab4/summary.md
|
||
|
||
Run from the repository root with the stand up. Tokens, signed URLs and secrets
|
||
are never written to the output.
|
||
"""
|
||
import argparse
|
||
import base64
|
||
import hashlib
|
||
import http.client
|
||
import json
|
||
import os
|
||
import statistics
|
||
import subprocess
|
||
import sys
|
||
import threading
|
||
import time
|
||
import urllib.parse
|
||
import uuid
|
||
from datetime import datetime, timezone
|
||
|
||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||
from pngkit import noise_png # noqa: E402
|
||
|
||
|
||
def sh(*args, check=True):
|
||
return subprocess.run(args, check=check, capture_output=True, text=True).stdout.strip()
|
||
|
||
|
||
def read_env(path=".env"):
|
||
values = {}
|
||
with open(path) as handle:
|
||
for line in handle:
|
||
line = line.strip()
|
||
if line and not line.startswith("#") and "=" in line:
|
||
key, _, value = line.partition("=")
|
||
values[key] = value
|
||
return values
|
||
|
||
|
||
def request(host, port, method, path, headers=None, body=None):
|
||
connection = http.client.HTTPConnection(host, port, timeout=120)
|
||
try:
|
||
connection.request(method, path, body=body, headers=headers or {})
|
||
response = connection.getresponse()
|
||
payload = response.read()
|
||
return response.status, payload
|
||
finally:
|
||
connection.close()
|
||
|
||
|
||
def api_json(env, method, path, token=None, body=None, basic=None):
|
||
headers = {}
|
||
data = None
|
||
if body is not None:
|
||
headers["content-type"] = "application/json"
|
||
data = json.dumps(body).encode()
|
||
if token:
|
||
headers["authorization"] = f"Bearer {token}"
|
||
if basic:
|
||
headers["authorization"] = "Basic " + base64.b64encode(basic.encode()).decode()
|
||
port = int(env.get("APP_PUBLIC_PORT", "8080"))
|
||
status, payload = request("127.0.0.1", port, method, path, headers, data)
|
||
return status, json.loads(payload) if payload else None
|
||
|
||
|
||
def containers():
|
||
ids = sh("docker", "compose", "--env-file", ".env", "ps", "-q", "api").split()
|
||
return sorted(ids)
|
||
|
||
|
||
NODE_METRICS = (
|
||
"fetch('http://127.0.0.1:'+(process.env.APP_PORT||8080)+'/internal/metrics'+process.argv[1],"
|
||
"{method:process.argv[1]?'POST':'GET',headers:{authorization:'Bearer '+process.env.METRICS_TOKEN}})"
|
||
".then(r=>r.text()).then(console.log)"
|
||
)
|
||
|
||
|
||
def metrics(container):
|
||
return json.loads(sh("docker", "exec", container, "node", "-e", NODE_METRICS, ""))
|
||
|
||
|
||
def reset_metrics(container):
|
||
sh("docker", "exec", container, "node", "-e", NODE_METRICS, "/reset")
|
||
|
||
|
||
def net_bytes(container):
|
||
for line in sh("docker", "exec", container, "cat", "/proc/net/dev").splitlines():
|
||
if line.strip().startswith("eth0:"):
|
||
fields = line.split(":", 1)[1].split()
|
||
return int(fields[0]), int(fields[8])
|
||
raise RuntimeError("eth0 not found")
|
||
|
||
|
||
def percentile(values, fraction):
|
||
ordered = sorted(values)
|
||
if not ordered:
|
||
return 0.0
|
||
return ordered[min(len(ordered) - 1, max(0, int(round(fraction * len(ordered) + 0.5)) - 1))]
|
||
|
||
|
||
def one_upload(env, mode, token, chat_id, data, seed):
|
||
"""Returns (ok, elapsed_ms, outcome); outcome names the failing step and status, e.g. "complete:429"."""
|
||
digest = hashlib.sha256(data).hexdigest()
|
||
filename = f"bench-{seed}.png"
|
||
port = int(env.get("APP_PUBLIC_PORT", "8080"))
|
||
started = time.perf_counter()
|
||
outcome, attachment = "ok", None
|
||
try:
|
||
if mode == "proxy":
|
||
status, body = request(
|
||
"127.0.0.1", port, "POST",
|
||
f"/api/v1/chats/{chat_id}/attachments?filename={urllib.parse.quote(filename)}",
|
||
{"authorization": f"Bearer {token}", "content-type": "image/png"}, data,
|
||
)
|
||
if status == 201:
|
||
attachment = json.loads(body)
|
||
else:
|
||
outcome = f"upload:{status}"
|
||
else:
|
||
headers = {"authorization": f"Bearer {token}", "content-type": "application/json"}
|
||
status, body = request(
|
||
"127.0.0.1", port, "POST", f"/api/v1/chats/{chat_id}/attachments/direct-uploads", headers,
|
||
json.dumps({"filename": filename, "content_type": "image/png",
|
||
"size_bytes": len(data), "sha256": digest}).encode(),
|
||
)
|
||
if status != 201:
|
||
outcome = f"init:{status}"
|
||
else:
|
||
ticket = json.loads(body)
|
||
target = urllib.parse.urlsplit(ticket["url"])
|
||
put_status, _ = request(
|
||
target.hostname, target.port, "PUT", target.path + "?" + target.query,
|
||
{**ticket["headers"]}, data,
|
||
)
|
||
if put_status != 200:
|
||
outcome = f"put:{put_status}"
|
||
else:
|
||
status, body = request(
|
||
"127.0.0.1", port, "POST", f"/api/v1/attachments/{ticket['attachment_id']}/complete",
|
||
{"authorization": f"Bearer {token}"},
|
||
)
|
||
if status == 200:
|
||
attachment = json.loads(body)
|
||
else:
|
||
outcome = f"complete:{status}"
|
||
except (OSError, http.client.HTTPException, ValueError) as error:
|
||
outcome = f"error:{type(error).__name__}"
|
||
elapsed_ms = (time.perf_counter() - started) * 1000
|
||
if outcome == "ok" and (attachment is None or attachment["sha256"] != digest):
|
||
outcome = "checksum_mismatch"
|
||
return outcome == "ok", elapsed_ms, outcome
|
||
|
||
|
||
def payload(args, seed):
|
||
if args.image:
|
||
with open(args.image, "rb") as handle:
|
||
return handle.read()
|
||
return noise_png(args.size_bytes, seed)
|
||
|
||
|
||
def run_once(env, args, actors, ids):
|
||
for container in ids:
|
||
reset_metrics(container)
|
||
time.sleep(0.2)
|
||
baseline = {c: metrics(c) for c in ids}
|
||
net_before = {c: net_bytes(c) for c in ids}
|
||
latencies, failures = [], []
|
||
outcomes = {}
|
||
lock = threading.Lock()
|
||
|
||
def worker(index):
|
||
for step in range(args.uploads_per_worker):
|
||
seed = index * 1000 + step
|
||
data = payload(args, seed)
|
||
token, chat_id = actors[index % len(actors)]
|
||
ok, elapsed, outcome = one_upload(env, args.mode, token, chat_id, data, seed)
|
||
with lock:
|
||
(latencies if ok else failures).append(elapsed)
|
||
outcomes[outcome] = outcomes.get(outcome, 0) + 1
|
||
|
||
started = time.perf_counter()
|
||
threads = [threading.Thread(target=worker, args=(i,)) for i in range(args.concurrency)]
|
||
for thread in threads:
|
||
thread.start()
|
||
for thread in threads:
|
||
thread.join()
|
||
duration = time.perf_counter() - started
|
||
time.sleep(0.2)
|
||
after = {c: metrics(c) for c in ids}
|
||
net_after = {c: net_bytes(c) for c in ids}
|
||
|
||
replicas = []
|
||
for c in ids:
|
||
base_rss = baseline[c]["memory_bytes"]["rss"]
|
||
peak = after[c]["memory_peak_bytes"]
|
||
replicas.append({
|
||
"instance_id": after[c]["instance_id"],
|
||
"delta_rx_bytes": net_after[c][0] - net_before[c][0],
|
||
"delta_tx_bytes": net_after[c][1] - net_before[c][1],
|
||
"baseline_rss_bytes": base_rss,
|
||
"peak_memory_bytes": peak,
|
||
"delta_rss_bytes": peak["rss"] - base_rss,
|
||
"counters": {k: v for k, v in after[c]["counters"].items() if k.startswith("attachments.")},
|
||
})
|
||
total = args.concurrency * args.uploads_per_worker
|
||
return {
|
||
"uploads": total,
|
||
"succeeded": len(latencies),
|
||
"failed": len(failures),
|
||
"outcomes": outcomes,
|
||
"duration_s": round(duration, 3),
|
||
"latency_ms": {
|
||
"p50": round(percentile(latencies, 0.50), 1),
|
||
"p95": round(percentile(latencies, 0.95), 1),
|
||
"p99": round(percentile(latencies, 0.99), 1),
|
||
},
|
||
"replicas": replicas,
|
||
"total_delta_rx_bytes": sum(r["delta_rx_bytes"] for r in replicas),
|
||
"total_delta_tx_bytes": sum(r["delta_tx_bytes"] for r in replicas),
|
||
"rx_bytes_per_upload": round(sum(r["delta_rx_bytes"] for r in replicas) / total),
|
||
"tx_bytes_per_upload": round(sum(r["delta_tx_bytes"] for r in replicas) / total),
|
||
"max_delta_rss_bytes": max(r["delta_rss_bytes"] for r in replicas),
|
||
}
|
||
|
||
|
||
def command_run(args):
|
||
if args.image:
|
||
args.size_bytes = os.path.getsize(args.image)
|
||
if args.size_bytes <= 0:
|
||
sys.exit("--size-bytes or --image is required")
|
||
env = read_env()
|
||
ids = containers()
|
||
if not ids:
|
||
sys.exit("no api containers found")
|
||
# One user per worker: the API caps concurrent uploads per user
|
||
# (UPLOAD_MAX_IN_FLIGHT_PER_USER), and real traffic comes from many users.
|
||
actors = []
|
||
for _ in range(args.concurrency):
|
||
name = f"bench_{uuid.uuid4().hex[:10]}"
|
||
password = "bench:" + uuid.uuid4().hex
|
||
status, user = api_json(env, "POST", "/api/v1/users",
|
||
body={"username": name, "display_name": "bench", "password": password})
|
||
assert status == 201, status
|
||
status, session = api_json(env, "POST", "/api/v1/auth/sessions", basic=f"{name}:{password}")
|
||
assert status == 201, status
|
||
token = session["token"]
|
||
status, chat = api_json(env, "POST", "/api/v1/chats", token, {"title": "bench", "member_ids": [user["id"]]})
|
||
assert status == 201, status
|
||
actors.append((token, chat["id"]))
|
||
|
||
# Warm-up: two uploads per replica, not measured.
|
||
for seed in range(len(ids) * 2):
|
||
token, chat_id = actors[seed % len(actors)]
|
||
one_upload(env, args.mode, token, chat_id, payload(args, 900000 + seed), seed)
|
||
|
||
limits = []
|
||
for c in ids:
|
||
limits.append(sh("docker", "inspect", "-f", "{{.HostConfig.NanoCpus}} {{.HostConfig.Memory}}", c))
|
||
repeats = [run_once(env, args, actors, ids) for _ in range(args.repeat)]
|
||
mem_total = next(int(l.split()[1]) * 1024 for l in open("/proc/meminfo") if l.startswith("MemTotal"))
|
||
result = {
|
||
"date": datetime.now(timezone.utc).isoformat(timespec="seconds"),
|
||
"git": sh("git", "rev-parse", "--short", "HEAD", check=False),
|
||
"params": {
|
||
"mode": args.mode, "size_bytes": args.size_bytes, "concurrency": args.concurrency,
|
||
"uploads_per_worker": args.uploads_per_worker, "repeat": args.repeat,
|
||
"users": args.concurrency, "image": os.path.basename(args.image) if args.image else None,
|
||
},
|
||
"host": {"nproc": os.cpu_count(), "mem_total_bytes": mem_total},
|
||
"api_container_limits_nanocpus_memory": limits,
|
||
"repeats": repeats,
|
||
}
|
||
os.makedirs(os.path.dirname(os.path.abspath(args.output)), exist_ok=True)
|
||
with open(args.output, "w") as handle:
|
||
json.dump(result, handle, indent=2)
|
||
handle.write("\n")
|
||
for index, item in enumerate(repeats):
|
||
print(f"repeat {index + 1}: ok={item['succeeded']}/{item['uploads']} "
|
||
f"rx/upload={item['rx_bytes_per_upload']} tx/upload={item['tx_bytes_per_upload']} "
|
||
f"maxΔrss={item['max_delta_rss_bytes']} p50={item['latency_ms']['p50']}ms "
|
||
f"outcomes={item['outcomes']}")
|
||
|
||
|
||
def mib(value):
|
||
return f"{value / 1048576:.2f}"
|
||
|
||
|
||
def command_summarize(args):
|
||
rows = []
|
||
for path in args.paths:
|
||
with open(path) as handle:
|
||
data = json.load(handle)
|
||
p = data["params"]
|
||
reps = data["repeats"]
|
||
med = lambda key: statistics.median(r[key] for r in reps) # noqa: E731
|
||
label = p["mode"] + (f" ({p['image']})" if p.get("image") else "")
|
||
rows.append((label, p["size_bytes"], p["concurrency"], reps, med, data))
|
||
rows.sort(key=lambda r: (r[0].count("("), r[1], r[0], r[2]))
|
||
print("| mode | size, MiB | C | ok | failures | API rx/upload, MiB | API tx/upload, MiB | max Δrss, MiB | peak RSS, MiB | p50, ms | p95, ms |")
|
||
print("|---|---|---|---|---|---|---|---|---|---|---|")
|
||
for mode, size, conc, reps, med, _data in rows:
|
||
peak = max(x["peak_memory_bytes"]["rss"] for r in reps for x in r["replicas"])
|
||
failed = {}
|
||
for rep in reps:
|
||
for name, count in rep.get("outcomes", {}).items():
|
||
if name != "ok":
|
||
failed[name] = failed.get(name, 0) + count
|
||
failures = ", ".join(f"{name}×{count}" for name, count in sorted(failed.items())) or "—"
|
||
ok = sum(r["succeeded"] for r in reps)
|
||
total = sum(r["uploads"] for r in reps)
|
||
p50 = statistics.median(r["latency_ms"]["p50"] for r in reps)
|
||
p95 = statistics.median(r["latency_ms"]["p95"] for r in reps)
|
||
print(f"| {mode} | {mib(size)} | {conc} | {ok}/{total} | {failures} | {mib(med('rx_bytes_per_upload'))} | "
|
||
f"{mib(med('tx_bytes_per_upload'))} | {mib(med('max_delta_rss_bytes'))} | {mib(peak)} | {p50:.0f} | {p95:.0f} |")
|
||
|
||
|
||
def main():
|
||
parser = argparse.ArgumentParser()
|
||
sub = parser.add_subparsers(dest="command", required=True)
|
||
run = sub.add_parser("run")
|
||
run.add_argument("--mode", choices=["proxy", "direct"], required=True)
|
||
run.add_argument("--size-bytes", type=int, default=0)
|
||
run.add_argument("--image", help="upload this file instead of generated noise PNGs")
|
||
run.add_argument("--concurrency", type=int, default=10)
|
||
run.add_argument("--uploads-per-worker", type=int, default=5)
|
||
run.add_argument("--repeat", type=int, default=3)
|
||
run.add_argument("--output", required=True)
|
||
summarize = sub.add_parser("summarize")
|
||
summarize.add_argument("paths", nargs="+")
|
||
args = parser.parse_args()
|
||
{"run": command_run, "summarize": command_summarize}[args.command](args)
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|