#!/usr/bin/env python3 """Upload benchmark: API traffic and peak RSS per upload for proxy vs direct mode. attachments_bench.py run --mode proxy --size-bytes 1048576 --concurrency 10 \ --uploads-per-worker 5 --repeat 3 --output evidence/lab4/bench/proxy-1048576-c10.json attachments_bench.py summarize evidence/lab4/bench/*.json > evidence/lab4/summary.md Run from the repository root with the stand up. Tokens, signed URLs and secrets are never written to the output. """ import argparse import base64 import hashlib import http.client import json import os import statistics import subprocess import sys import threading import time import urllib.parse import uuid from datetime import datetime, timezone sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) from pngkit import noise_png # noqa: E402 def sh(*args, check=True): return subprocess.run(args, check=check, capture_output=True, text=True).stdout.strip() def read_env(path=".env"): values = {} with open(path) as handle: for line in handle: line = line.strip() if line and not line.startswith("#") and "=" in line: key, _, value = line.partition("=") values[key] = value return values def request(host, port, method, path, headers=None, body=None): connection = http.client.HTTPConnection(host, port, timeout=120) try: connection.request(method, path, body=body, headers=headers or {}) response = connection.getresponse() payload = response.read() return response.status, payload finally: connection.close() def api_json(env, method, path, token=None, body=None, basic=None): headers = {} data = None if body is not None: headers["content-type"] = "application/json" data = json.dumps(body).encode() if token: headers["authorization"] = f"Bearer {token}" if basic: headers["authorization"] = "Basic " + base64.b64encode(basic.encode()).decode() port = int(env.get("APP_PUBLIC_PORT", "8080")) status, payload = request("127.0.0.1", port, method, path, headers, data) return status, json.loads(payload) if payload else None def containers(): ids = sh("docker", "compose", "--env-file", ".env", "ps", "-q", "api").split() return sorted(ids) NODE_METRICS = ( "fetch('http://127.0.0.1:'+(process.env.APP_PORT||8080)+'/internal/metrics'+process.argv[1]," "{method:process.argv[1]?'POST':'GET',headers:{authorization:'Bearer '+process.env.METRICS_TOKEN}})" ".then(r=>r.text()).then(console.log)" ) def metrics(container): return json.loads(sh("docker", "exec", container, "node", "-e", NODE_METRICS, "")) def reset_metrics(container): sh("docker", "exec", container, "node", "-e", NODE_METRICS, "/reset") def net_bytes(container): for line in sh("docker", "exec", container, "cat", "/proc/net/dev").splitlines(): if line.strip().startswith("eth0:"): fields = line.split(":", 1)[1].split() return int(fields[0]), int(fields[8]) raise RuntimeError("eth0 not found") def percentile(values, fraction): ordered = sorted(values) if not ordered: return 0.0 return ordered[min(len(ordered) - 1, max(0, int(round(fraction * len(ordered) + 0.5)) - 1))] def one_upload(env, mode, token, chat_id, data, seed): """Returns (ok, elapsed_ms, outcome); outcome names the failing step and status, e.g. "complete:429".""" digest = hashlib.sha256(data).hexdigest() filename = f"bench-{seed}.png" port = int(env.get("APP_PUBLIC_PORT", "8080")) started = time.perf_counter() outcome, attachment = "ok", None try: if mode == "proxy": status, body = request( "127.0.0.1", port, "POST", f"/api/v1/chats/{chat_id}/attachments?filename={urllib.parse.quote(filename)}", {"authorization": f"Bearer {token}", "content-type": "image/png"}, data, ) if status == 201: attachment = json.loads(body) else: outcome = f"upload:{status}" else: headers = {"authorization": f"Bearer {token}", "content-type": "application/json"} status, body = request( "127.0.0.1", port, "POST", f"/api/v1/chats/{chat_id}/attachments/direct-uploads", headers, json.dumps({"filename": filename, "content_type": "image/png", "size_bytes": len(data), "sha256": digest}).encode(), ) if status != 201: outcome = f"init:{status}" else: ticket = json.loads(body) target = urllib.parse.urlsplit(ticket["url"]) put_status, _ = request( target.hostname, target.port, "PUT", target.path + "?" + target.query, {**ticket["headers"]}, data, ) if put_status != 200: outcome = f"put:{put_status}" else: status, body = request( "127.0.0.1", port, "POST", f"/api/v1/attachments/{ticket['attachment_id']}/complete", {"authorization": f"Bearer {token}"}, ) if status == 200: attachment = json.loads(body) else: outcome = f"complete:{status}" except (OSError, http.client.HTTPException, ValueError) as error: outcome = f"error:{type(error).__name__}" elapsed_ms = (time.perf_counter() - started) * 1000 if outcome == "ok" and (attachment is None or attachment["sha256"] != digest): outcome = "checksum_mismatch" return outcome == "ok", elapsed_ms, outcome def payload(args, seed): if args.image: with open(args.image, "rb") as handle: return handle.read() return noise_png(args.size_bytes, seed) def run_once(env, args, actors, ids): for container in ids: reset_metrics(container) time.sleep(0.2) baseline = {c: metrics(c) for c in ids} net_before = {c: net_bytes(c) for c in ids} latencies, failures = [], [] outcomes = {} lock = threading.Lock() def worker(index): for step in range(args.uploads_per_worker): seed = index * 1000 + step data = payload(args, seed) token, chat_id = actors[index % len(actors)] ok, elapsed, outcome = one_upload(env, args.mode, token, chat_id, data, seed) with lock: (latencies if ok else failures).append(elapsed) outcomes[outcome] = outcomes.get(outcome, 0) + 1 started = time.perf_counter() threads = [threading.Thread(target=worker, args=(i,)) for i in range(args.concurrency)] for thread in threads: thread.start() for thread in threads: thread.join() duration = time.perf_counter() - started time.sleep(0.2) after = {c: metrics(c) for c in ids} net_after = {c: net_bytes(c) for c in ids} replicas = [] for c in ids: base_rss = baseline[c]["memory_bytes"]["rss"] peak = after[c]["memory_peak_bytes"] replicas.append({ "instance_id": after[c]["instance_id"], "delta_rx_bytes": net_after[c][0] - net_before[c][0], "delta_tx_bytes": net_after[c][1] - net_before[c][1], "baseline_rss_bytes": base_rss, "peak_memory_bytes": peak, "delta_rss_bytes": peak["rss"] - base_rss, "counters": {k: v for k, v in after[c]["counters"].items() if k.startswith("attachments.")}, }) total = args.concurrency * args.uploads_per_worker return { "uploads": total, "succeeded": len(latencies), "failed": len(failures), "outcomes": outcomes, "duration_s": round(duration, 3), "latency_ms": { "p50": round(percentile(latencies, 0.50), 1), "p95": round(percentile(latencies, 0.95), 1), "p99": round(percentile(latencies, 0.99), 1), }, "replicas": replicas, "total_delta_rx_bytes": sum(r["delta_rx_bytes"] for r in replicas), "total_delta_tx_bytes": sum(r["delta_tx_bytes"] for r in replicas), "rx_bytes_per_upload": round(sum(r["delta_rx_bytes"] for r in replicas) / total), "tx_bytes_per_upload": round(sum(r["delta_tx_bytes"] for r in replicas) / total), "max_delta_rss_bytes": max(r["delta_rss_bytes"] for r in replicas), } def command_run(args): if args.image: args.size_bytes = os.path.getsize(args.image) if args.size_bytes <= 0: sys.exit("--size-bytes or --image is required") env = read_env() ids = containers() if not ids: sys.exit("no api containers found") # One user per worker: the API caps concurrent uploads per user # (UPLOAD_MAX_IN_FLIGHT_PER_USER), and real traffic comes from many users. actors = [] for _ in range(args.concurrency): name = f"bench_{uuid.uuid4().hex[:10]}" password = "bench:" + uuid.uuid4().hex status, user = api_json(env, "POST", "/api/v1/users", body={"username": name, "display_name": "bench", "password": password}) assert status == 201, status status, session = api_json(env, "POST", "/api/v1/auth/sessions", basic=f"{name}:{password}") assert status == 201, status token = session["token"] status, chat = api_json(env, "POST", "/api/v1/chats", token, {"title": "bench", "member_ids": [user["id"]]}) assert status == 201, status actors.append((token, chat["id"])) # Warm-up: two uploads per replica, not measured. for seed in range(len(ids) * 2): token, chat_id = actors[seed % len(actors)] one_upload(env, args.mode, token, chat_id, payload(args, 900000 + seed), seed) limits = [] for c in ids: limits.append(sh("docker", "inspect", "-f", "{{.HostConfig.NanoCpus}} {{.HostConfig.Memory}}", c)) repeats = [run_once(env, args, actors, ids) for _ in range(args.repeat)] mem_total = next(int(l.split()[1]) * 1024 for l in open("/proc/meminfo") if l.startswith("MemTotal")) result = { "date": datetime.now(timezone.utc).isoformat(timespec="seconds"), "git": sh("git", "rev-parse", "--short", "HEAD", check=False), "params": { "mode": args.mode, "size_bytes": args.size_bytes, "concurrency": args.concurrency, "uploads_per_worker": args.uploads_per_worker, "repeat": args.repeat, "users": args.concurrency, "image": os.path.basename(args.image) if args.image else None, }, "host": {"nproc": os.cpu_count(), "mem_total_bytes": mem_total}, "api_container_limits_nanocpus_memory": limits, "repeats": repeats, } os.makedirs(os.path.dirname(os.path.abspath(args.output)), exist_ok=True) with open(args.output, "w") as handle: json.dump(result, handle, indent=2) handle.write("\n") for index, item in enumerate(repeats): print(f"repeat {index + 1}: ok={item['succeeded']}/{item['uploads']} " f"rx/upload={item['rx_bytes_per_upload']} tx/upload={item['tx_bytes_per_upload']} " f"maxΔrss={item['max_delta_rss_bytes']} p50={item['latency_ms']['p50']}ms " f"outcomes={item['outcomes']}") def mib(value): return f"{value / 1048576:.2f}" def command_summarize(args): rows = [] for path in args.paths: with open(path) as handle: data = json.load(handle) p = data["params"] reps = data["repeats"] med = lambda key: statistics.median(r[key] for r in reps) # noqa: E731 label = p["mode"] + (f" ({p['image']})" if p.get("image") else "") rows.append((label, p["size_bytes"], p["concurrency"], reps, med, data)) rows.sort(key=lambda r: (r[0].count("("), r[1], r[0], r[2])) print("| mode | size, MiB | C | ok | failures | API rx/upload, MiB | API tx/upload, MiB | max Δrss, MiB | peak RSS, MiB | p50, ms | p95, ms |") print("|---|---|---|---|---|---|---|---|---|---|---|") for mode, size, conc, reps, med, _data in rows: peak = max(x["peak_memory_bytes"]["rss"] for r in reps for x in r["replicas"]) failed = {} for rep in reps: for name, count in rep.get("outcomes", {}).items(): if name != "ok": failed[name] = failed.get(name, 0) + count failures = ", ".join(f"{name}×{count}" for name, count in sorted(failed.items())) or "—" ok = sum(r["succeeded"] for r in reps) total = sum(r["uploads"] for r in reps) p50 = statistics.median(r["latency_ms"]["p50"] for r in reps) p95 = statistics.median(r["latency_ms"]["p95"] for r in reps) print(f"| {mode} | {mib(size)} | {conc} | {ok}/{total} | {failures} | {mib(med('rx_bytes_per_upload'))} | " f"{mib(med('tx_bytes_per_upload'))} | {mib(med('max_delta_rss_bytes'))} | {mib(peak)} | {p50:.0f} | {p95:.0f} |") def main(): parser = argparse.ArgumentParser() sub = parser.add_subparsers(dest="command", required=True) run = sub.add_parser("run") run.add_argument("--mode", choices=["proxy", "direct"], required=True) run.add_argument("--size-bytes", type=int, default=0) run.add_argument("--image", help="upload this file instead of generated noise PNGs") run.add_argument("--concurrency", type=int, default=10) run.add_argument("--uploads-per-worker", type=int, default=5) run.add_argument("--repeat", type=int, default=3) run.add_argument("--output", required=True) summarize = sub.add_parser("summarize") summarize.add_argument("paths", nargs="+") args = parser.parse_args() {"run": command_run, "summarize": command_summarize}[args.command](args) if __name__ == "__main__": main()