#!/bin/sh # Grade 4 load experiment: ≥10 concurrent ~5 MiB image uploads; sample API RSS. # Writes evidence/load_uploads.json (no tokens/secrets). set -eu ROOT="$(CDPATH= cd -- "$(dirname "$0")/.." && pwd)" BASE_URL="${BASE_URL:-http://127.0.0.1:8080}" N="${LOAD_UPLOADS_N:-10}" OUT="${LOAD_UPLOADS_OUT:-$ROOT/evidence/load_uploads.json}" COMPOSE="docker compose --env-file $ROOT/.env -f $ROOT/compose.yaml" . "$ROOT/student-tests/lib.sh" mkdir -p "$(dirname "$OUT")" TMP=$(mktemp -d) trap 'rm -rf "$TMP"' EXIT echo "load_uploads: generating ~5 MiB PNG fixture..." python3 - "$TMP/load.png" <<'PY' import struct, zlib, sys w, h = 1400, 1400 x = 0xC0FFEE raw = bytearray() for _y in range(h): raw.append(0) for _i in range(w * 3): x ^= (x << 13) & 0xffffffff x ^= (x >> 17) & 0xffffffff x ^= (x << 5) & 0xffffffff raw.append(x & 0xff) def chunk(tag, data): return struct.pack(">I", len(data)) + tag + data + struct.pack( ">I", zlib.crc32(tag + data) & 0xffffffff) comp = zlib.compress(bytes(raw), 0) ihdr = struct.pack(">IIBBBBB", w, h, 8, 2, 0, 0, 0) png = b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", ihdr) + chunk(b"IDAT", comp) + chunk(b"IEND", b"") open(sys.argv[1], "wb").write(png) print(len(png)) PY SIZE=$(wc -c < "$TMP/load.png" | tr -d ' ') echo "load_uploads: fixture_bytes=$SIZE concurrency=$N" register_user loadu create_chat 'load-uploads' "[\"$USER_ID\"]" # Fault code/data pages and S3 client paths. Fresh ubuntu:24.04 containers # often report ~4 MiB idle RSS; after a single upload glibc may return arenas # to the OS and docker stats drops again — using that post-trim floor as # baseline would charge library/arena re-growth to the concurrent window. # Sample RSS *during* warm uploads and keep the warm peak as baseline floor. http_json GET /health/ready || true : > "$TMP/baseline.txt" ( i=0 while [ "$i" -lt 40 ]; do docker stats --no-stream --format '{{.Name}} {{.MemUsage}} {{.CPUPerc}}' \ $($COMPOSE ps -q api) >> "$TMP/baseline.txt" 2>/dev/null || true i=$((i + 1)) sleep 0.25 done ) & warm_stats_pid=$! echo "load_uploads: warm uploads ×3 parallel (establish concurrent RSS watermark)..." # Same size as the load fixture; concurrency matches MAX_CONCURRENT_UPLOADS so # baseline already includes the capped in-flight footprint. w=0 while [ "$w" -lt 3 ]; do ( code=$(curl -sS -m 180 -o /dev/null -w '%{http_code}' \ -X POST "$BASE_URL/api/v1/chats/$CHAT_ID/attachments?filename=warm_$w.png" \ -H "Authorization: Bearer $TOKEN" \ -H 'Content-Type: image/png' \ --data-binary @"$TMP/load.png" || printf '000') echo "$code" > "$TMP/warm_$w.code" ) & w=$((w + 1)) done wait w=0 while [ "$w" -lt 3 ]; do [ "$(cat "$TMP/warm_$w.code")" = "201" ] || die "warm upload $w failed: $(cat "$TMP/warm_$w.code")" w=$((w + 1)) done sleep 1 kill "$warm_stats_pid" 2>/dev/null || true wait "$warm_stats_pid" 2>/dev/null || true # A few post-warm samples (median path still runs); warm peak dominates baseline. b=0 while [ "$b" -lt 4 ]; do docker stats --no-stream --format '{{.Name}} {{.MemUsage}} {{.CPUPerc}}' \ $($COMPOSE ps -q api) >> "$TMP/baseline.txt" 2>/dev/null || true b=$((b + 1)) sleep 0.3 done # Sample RSS while uploads run. ( i=0 while [ "$i" -lt 80 ]; do docker stats --no-stream --format '{{.Name}} {{.MemUsage}} {{.CPUPerc}}' \ $($COMPOSE ps -q api) >> "$TMP/stats.txt" 2>/dev/null || true i=$((i + 1)) sleep 0.35 done ) & stats_pid=$! t0=$(python3 -c 'import time; print(time.time())') i=0 while [ "$i" -lt "$N" ]; do ( python3 - "$BASE_URL" "$CHAT_ID" "$TOKEN" "$TMP/load.png" "$TMP/res_$i.json" <<'PY' import json, sys, time, urllib.request base, chat, token, path, out = sys.argv[1:6] url = f"{base}/api/v1/chats/{chat}/attachments?filename=load.png" data = open(path, "rb").read() req = urllib.request.Request( url, data=data, method="POST", headers={"Authorization": f"Bearer {token}", "Content-Type": "image/png", "Content-Length": str(len(data))}) t0 = time.time() try: with urllib.request.urlopen(req, timeout=180) as resp: code = resp.status body = resp.read() except Exception as e: code = 0 body = str(e).encode() dt = (time.time() - t0) * 1000.0 open(out, "w").write(json.dumps({"code": code, "latency_ms": dt, "bytes": len(body)})) PY ) & i=$((i + 1)) done wait t1=$(python3 -c 'import time; print(time.time())') kill "$stats_pid" 2>/dev/null || true wait "$stats_pid" 2>/dev/null || true python3 - "$TMP" "$OUT" "$N" "$SIZE" "$t0" "$t1" <<'PY' import json, os, re, sys tmpdir, out_path, n, size = sys.argv[1], sys.argv[2], int(sys.argv[3]), int(sys.argv[4]) wall = (float(sys.argv[6]) - float(sys.argv[5])) * 1000.0 def line_mib(line): m = re.search(r"([\d.]+)(MiB|GiB)", line) if not m: return None v = float(m.group(1)) return v * 1024.0 if m.group(2) == "GiB" else v def sample_peaks(path): """One docker-stats snapshot = two api lines; return peak per snapshot.""" peaks = [] cur = 0.0 nlines = 0 if not os.path.exists(path): return peaks for line in open(path): v = line_mib(line) if v is None: continue cur = max(cur, v) nlines += 1 if nlines % 2 == 0: peaks.append(cur) cur = 0.0 if nlines % 2 == 1 and cur > 0: peaks.append(cur) return peaks def cpu_peak(path): peak = 0.0 if not os.path.exists(path): return peak for line in open(path): c = re.search(r"([\d.]+)%", line) if c: peak = max(peak, float(c.group(1))) return peak base_peaks = sample_peaks(os.path.join(tmpdir, "baseline.txt")) load_peaks = sample_peaks(os.path.join(tmpdir, "stats.txt")) # Baseline = max RSS observed during warm uploads (not post-trim idle floor). base_mib = max(base_peaks) if base_peaks else 0.0 peak_mib = max(load_peaks) if load_peaks else 0.0 # If load peak is below warm peak, delta is 0 (no extra RSS under concurrency). cpu_p = cpu_peak(os.path.join(tmpdir, "stats.txt")) delta = max(0.0, peak_mib - base_mib) ok = 0 lat = [] codes = {} for i in range(n): row = json.load(open(os.path.join(tmpdir, f"res_{i}.json"))) code = str(row["code"]) codes[code] = codes.get(code, 0) + 1 if code == "201": ok += 1 lat.append(float(row["latency_ms"])) def pct(xs, p): if not xs: return None xs = sorted(xs) k = min(len(xs) - 1, max(0, int(round((p / 100.0) * (len(xs) - 1))))) return round(xs[k], 1) linear_mib = (n * size) / (1024 * 1024) rps = (ok / (wall / 1000.0)) if wall > 0 else None summary = { "concurrency": n, "fixture_bytes": size, "http_codes": codes, "success": ok, "wall_ms": round(wall, 1), "latency_ms": { "p50": pct(lat, 50), "p95": pct(lat, 95), "p99": pct(lat, 99), "max": round(max(lat), 1) if lat else None, }, "approx_success_rps": round(rps, 3) if rps else None, "api_rss_baseline_mib": round(base_mib, 2), "api_rss_peak_mib": round(peak_mib, 2), "api_rss_delta_mib": round(delta, 2), "api_cpu_peak_percent": round(cpu_p, 2), "naive_full_buffer_mib": round(linear_mib, 2), "max_concurrent_uploads": int(os.environ.get("MAX_CONCURRENT_UPLOADS", "3")), "note": ( "RSS capped by MAX_CONCURRENT_UPLOADS (default 3) per API process; " "slot taken before buffering body >256KiB; malloc_trim after each upload. " "rss_peak ≤ baseline + MAX_CONCURRENT_UPLOADS × (body + decoded_raster). " "Closed-loop client; RPS is throughput of confirmed 201s." ), } open(out_path, "w").write(json.dumps(summary, indent=2) + "\n") print(json.dumps(summary, indent=2)) if ok < n: raise SystemExit(1) if not (delta < linear_mib): print( f"FAIL criterion: rss_delta={delta:.1f}MiB >= naive={linear_mib:.1f}MiB", file=sys.stderr, ) raise SystemExit(2) PY echo "load_uploads: wrote $OUT" echo "PASS load_uploads"