#!/usr/bin/env python3 """Upload benchmark: API traffic and peak RSS per upload for proxy vs direct mode. attachments_bench.py run --mode proxy --size-bytes 1048576 --concurrency 10 \ --uploads-per-worker 5 --repeat 3 --output evidence/lab4/bench/proxy-1048576-c10.json attachments_bench.py summarize evidence/lab4/bench/*.json > evidence/lab4/summary.md Run from the repository root with the stand up. Tokens, signed URLs and secrets are never written to the output. """ import argparse import base64 import hashlib import http.client import json import os import statistics import subprocess import sys import threading import time import urllib.parse import uuid from datetime import datetime, timezone sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) from pngkit import noise_png # noqa: E402 def sh(*args, check=True): return subprocess.run(args, check=check, capture_output=True, text=True).stdout.strip() def read_env(path=".env"): values = {} with open(path) as handle: for line in handle: line = line.strip() if line and not line.startswith("#") and "=" in line: key, _, value = line.partition("=") values[key] = value return values def request(host, port, method, path, headers=None, body=None): connection = http.client.HTTPConnection(host, port, timeout=120) try: connection.request(method, path, body=body, headers=headers or {}) response = connection.getresponse() payload = response.read() return response.status, payload finally: connection.close() def api_json(env, method, path, token=None, body=None, basic=None): headers = {} data = None if body is not None: headers["content-type"] = "application/json" data = json.dumps(body).encode() if token: headers["authorization"] = f"Bearer {token}" if basic: headers["authorization"] = "Basic " + base64.b64encode(basic.encode()).decode() port = int(env.get("APP_PUBLIC_PORT", "8080")) status, payload = request("127.0.0.1", port, method, path, headers, data) return status, json.loads(payload) if payload else None def containers(): ids = sh("docker", "compose", "--env-file", ".env", "ps", "-q", "api").split() return sorted(ids) NODE_METRICS = ( "fetch('http://127.0.0.1:'+(process.env.APP_PORT||8080)+'/internal/metrics'+process.argv[1]," "{method:process.argv[1]?'POST':'GET',headers:{authorization:'Bearer '+process.env.METRICS_TOKEN}})" ".then(r=>r.text()).then(console.log)" ) def metrics(container): return json.loads(sh("docker", "exec", container, "node", "-e", NODE_METRICS, "")) def reset_metrics(container): sh("docker", "exec", container, "node", "-e", NODE_METRICS, "/reset") def net_bytes(container): for line in sh("docker", "exec", container, "cat", "/proc/net/dev").splitlines(): if line.strip().startswith("eth0:"): fields = line.split(":", 1)[1].split() return int(fields[0]), int(fields[8]) raise RuntimeError("eth0 not found") def percentile(values, fraction): ordered = sorted(values) if not ordered: return 0.0 return ordered[min(len(ordered) - 1, max(0, int(round(fraction * len(ordered) + 0.5)) - 1))] def one_upload(env, mode, token, chat_id, data, seed): digest = hashlib.sha256(data).hexdigest() filename = f"bench-{seed}.png" port = int(env.get("APP_PUBLIC_PORT", "8080")) started = time.perf_counter() if mode == "proxy": status, payload = request( "127.0.0.1", port, "POST", f"/api/v1/chats/{chat_id}/attachments?filename={urllib.parse.quote(filename)}", {"authorization": f"Bearer {token}", "content-type": "image/png"}, data, ) ok = status == 201 attachment = json.loads(payload) if ok else None else: status, ticket = api_json( env, "POST", f"/api/v1/chats/{chat_id}/attachments/direct-uploads", token, {"filename": filename, "content_type": "image/png", "size_bytes": len(data), "sha256": digest}, ) ok = status == 201 attachment = None if ok: target = urllib.parse.urlsplit(ticket["url"]) put_status, _ = request( target.hostname, target.port, "PUT", target.path + "?" + target.query, {**ticket["headers"]}, data, ) ok = put_status == 200 if ok: status, attachment = api_json( env, "POST", f"/api/v1/attachments/{ticket['attachment_id']}/complete", token ) ok = status == 200 elapsed_ms = (time.perf_counter() - started) * 1000 if ok and (attachment is None or attachment["sha256"] != digest): ok = False return ok, elapsed_ms def run_once(env, args, token, chat_id, ids): for container in ids: reset_metrics(container) time.sleep(0.2) baseline = {c: metrics(c) for c in ids} net_before = {c: net_bytes(c) for c in ids} latencies, failures = [], [] lock = threading.Lock() def worker(index): for step in range(args.uploads_per_worker): seed = index * 1000 + step data = noise_png(args.size_bytes, seed) ok, elapsed = one_upload(env, args.mode, token, chat_id, data, seed) with lock: (latencies if ok else failures).append(elapsed) started = time.perf_counter() threads = [threading.Thread(target=worker, args=(i,)) for i in range(args.concurrency)] for thread in threads: thread.start() for thread in threads: thread.join() duration = time.perf_counter() - started time.sleep(0.2) after = {c: metrics(c) for c in ids} net_after = {c: net_bytes(c) for c in ids} replicas = [] for c in ids: base_rss = baseline[c]["memory_bytes"]["rss"] peak = after[c]["memory_peak_bytes"] replicas.append({ "instance_id": after[c]["instance_id"], "delta_rx_bytes": net_after[c][0] - net_before[c][0], "delta_tx_bytes": net_after[c][1] - net_before[c][1], "baseline_rss_bytes": base_rss, "peak_memory_bytes": peak, "delta_rss_bytes": peak["rss"] - base_rss, "counters": {k: v for k, v in after[c]["counters"].items() if k.startswith("attachments.")}, }) total = args.concurrency * args.uploads_per_worker return { "uploads": total, "succeeded": len(latencies), "failed": len(failures), "duration_s": round(duration, 3), "latency_ms": { "p50": round(percentile(latencies, 0.50), 1), "p95": round(percentile(latencies, 0.95), 1), "p99": round(percentile(latencies, 0.99), 1), }, "replicas": replicas, "total_delta_rx_bytes": sum(r["delta_rx_bytes"] for r in replicas), "total_delta_tx_bytes": sum(r["delta_tx_bytes"] for r in replicas), "rx_bytes_per_upload": round(sum(r["delta_rx_bytes"] for r in replicas) / total), "tx_bytes_per_upload": round(sum(r["delta_tx_bytes"] for r in replicas) / total), "max_delta_rss_bytes": max(r["delta_rss_bytes"] for r in replicas), } def command_run(args): env = read_env() ids = containers() if not ids: sys.exit("no api containers found") name = f"bench_{uuid.uuid4().hex[:10]}" password = "bench:" + uuid.uuid4().hex status, user = api_json(env, "POST", "/api/v1/users", body={"username": name, "display_name": "bench", "password": password}) assert status == 201, status status, session = api_json(env, "POST", "/api/v1/auth/sessions", basic=f"{name}:{password}") assert status == 201, status token = session["token"] status, chat = api_json(env, "POST", "/api/v1/chats", token, {"title": "bench", "member_ids": [user["id"]]}) assert status == 201, status # Warm-up: one upload per replica path, not measured. for seed in range(len(ids) * 2): one_upload(env, args.mode, token, chat["id"], noise_png(args.size_bytes, 900000 + seed), seed) limits = [] for c in ids: limits.append(sh("docker", "inspect", "-f", "{{.HostConfig.NanoCpus}} {{.HostConfig.Memory}}", c)) repeats = [run_once(env, args, token, chat["id"], ids) for _ in range(args.repeat)] mem_total = next(int(l.split()[1]) * 1024 for l in open("/proc/meminfo") if l.startswith("MemTotal")) result = { "date": datetime.now(timezone.utc).isoformat(timespec="seconds"), "git": sh("git", "rev-parse", "--short", "HEAD", check=False), "params": { "mode": args.mode, "size_bytes": args.size_bytes, "concurrency": args.concurrency, "uploads_per_worker": args.uploads_per_worker, "repeat": args.repeat, }, "host": {"nproc": os.cpu_count(), "mem_total_bytes": mem_total}, "api_container_limits_nanocpus_memory": limits, "repeats": repeats, } os.makedirs(os.path.dirname(os.path.abspath(args.output)), exist_ok=True) with open(args.output, "w") as handle: json.dump(result, handle, indent=2) handle.write("\n") for index, item in enumerate(repeats): print(f"repeat {index + 1}: ok={item['succeeded']}/{item['uploads']} " f"rx/upload={item['rx_bytes_per_upload']} tx/upload={item['tx_bytes_per_upload']} " f"maxΔrss={item['max_delta_rss_bytes']} p50={item['latency_ms']['p50']}ms") def mib(value): return f"{value / 1048576:.2f}" def command_summarize(args): rows = [] for path in args.paths: with open(path) as handle: data = json.load(handle) p = data["params"] reps = data["repeats"] med = lambda key: statistics.median(r[key] for r in reps) # noqa: E731 rows.append((p["mode"], p["size_bytes"], p["concurrency"], reps, med, data)) rows.sort(key=lambda r: (r[1], r[0], r[2])) print("| mode | size, MiB | C | ok | API rx/upload, MiB | API tx/upload, MiB | max Δrss, MiB | p50, ms | p95, ms |") print("|---|---|---|---|---|---|---|---|---|") for mode, size, conc, reps, med, _data in rows: ok = sum(r["succeeded"] for r in reps) total = sum(r["uploads"] for r in reps) p50 = statistics.median(r["latency_ms"]["p50"] for r in reps) p95 = statistics.median(r["latency_ms"]["p95"] for r in reps) print(f"| {mode} | {mib(size)} | {conc} | {ok}/{total} | {mib(med('rx_bytes_per_upload'))} | " f"{mib(med('tx_bytes_per_upload'))} | {mib(med('max_delta_rss_bytes'))} | {p50:.0f} | {p95:.0f} |") def main(): parser = argparse.ArgumentParser() sub = parser.add_subparsers(dest="command", required=True) run = sub.add_parser("run") run.add_argument("--mode", choices=["proxy", "direct"], required=True) run.add_argument("--size-bytes", type=int, required=True) run.add_argument("--concurrency", type=int, default=10) run.add_argument("--uploads-per-worker", type=int, default=5) run.add_argument("--repeat", type=int, default=3) run.add_argument("--output", required=True) summarize = sub.add_parser("summarize") summarize.add_argument("paths", nargs="+") args = parser.parse_args() {"run": command_run, "summarize": command_summarize}[args.command](args) if __name__ == "__main__": main()