Files
mtusi2026lab4/scripts/load_uploads.sh
T
2026-10-30 21:09:56 +03:00

255 lines
7.9 KiB
Bash
Executable File
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/bin/sh
# Grade 4 load experiment: ≥10 concurrent ~5 MiB image uploads; sample API RSS.
# Writes evidence/load_uploads.json (no tokens/secrets).
set -eu
ROOT="$(CDPATH= cd -- "$(dirname "$0")/.." && pwd)"
BASE_URL="${BASE_URL:-http://127.0.0.1:8080}"
N="${LOAD_UPLOADS_N:-10}"
OUT="${LOAD_UPLOADS_OUT:-$ROOT/evidence/load_uploads.json}"
COMPOSE="docker compose --env-file $ROOT/.env -f $ROOT/compose.yaml"
. "$ROOT/student-tests/lib.sh"
mkdir -p "$(dirname "$OUT")"
TMP=$(mktemp -d)
trap 'rm -rf "$TMP"' EXIT
echo "load_uploads: generating ~5 MiB PNG fixture..."
python3 - "$TMP/load.png" <<'PY'
import struct, zlib, sys
w, h = 1400, 1400
x = 0xC0FFEE
raw = bytearray()
for _y in range(h):
raw.append(0)
for _i in range(w * 3):
x ^= (x << 13) & 0xffffffff
x ^= (x >> 17) & 0xffffffff
x ^= (x << 5) & 0xffffffff
raw.append(x & 0xff)
def chunk(tag, data):
return struct.pack(">I", len(data)) + tag + data + struct.pack(
">I", zlib.crc32(tag + data) & 0xffffffff)
comp = zlib.compress(bytes(raw), 0)
ihdr = struct.pack(">IIBBBBB", w, h, 8, 2, 0, 0, 0)
png = b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", ihdr) + chunk(b"IDAT", comp) + chunk(b"IEND", b"")
open(sys.argv[1], "wb").write(png)
print(len(png))
PY
SIZE=$(wc -c < "$TMP/load.png" | tr -d ' ')
echo "load_uploads: fixture_bytes=$SIZE concurrency=$N"
register_user loadu
create_chat 'load-uploads' "[\"$USER_ID\"]"
# Fault code/data pages and S3 client paths. Fresh ubuntu:24.04 containers
# often report ~4 MiB idle RSS; after a single upload glibc may return arenas
# to the OS and docker stats drops again — using that post-trim floor as
# baseline would charge library/arena re-growth to the concurrent window.
# Sample RSS *during* warm uploads and keep the warm peak as baseline floor.
http_json GET /health/ready || true
: > "$TMP/baseline.txt"
(
i=0
while [ "$i" -lt 40 ]; do
docker stats --no-stream --format '{{.Name}} {{.MemUsage}} {{.CPUPerc}}' \
$($COMPOSE ps -q api) >> "$TMP/baseline.txt" 2>/dev/null || true
i=$((i + 1))
sleep 0.25
done
) &
warm_stats_pid=$!
echo "load_uploads: warm uploads ×3 parallel (establish concurrent RSS watermark)..."
# Same size as the load fixture; concurrency matches MAX_CONCURRENT_UPLOADS so
# baseline already includes the capped in-flight footprint.
w=0
while [ "$w" -lt 3 ]; do
(
code=$(curl -sS -m 180 -o /dev/null -w '%{http_code}' \
-X POST "$BASE_URL/api/v1/chats/$CHAT_ID/attachments?filename=warm_$w.png" \
-H "Authorization: Bearer $TOKEN" \
-H 'Content-Type: image/png' \
--data-binary @"$TMP/load.png" || printf '000')
echo "$code" > "$TMP/warm_$w.code"
) &
w=$((w + 1))
done
wait
w=0
while [ "$w" -lt 3 ]; do
[ "$(cat "$TMP/warm_$w.code")" = "201" ] || die "warm upload $w failed: $(cat "$TMP/warm_$w.code")"
w=$((w + 1))
done
sleep 1
kill "$warm_stats_pid" 2>/dev/null || true
wait "$warm_stats_pid" 2>/dev/null || true
# A few post-warm samples (median path still runs); warm peak dominates baseline.
b=0
while [ "$b" -lt 4 ]; do
docker stats --no-stream --format '{{.Name}} {{.MemUsage}} {{.CPUPerc}}' \
$($COMPOSE ps -q api) >> "$TMP/baseline.txt" 2>/dev/null || true
b=$((b + 1))
sleep 0.3
done
# Sample RSS while uploads run.
(
i=0
while [ "$i" -lt 80 ]; do
docker stats --no-stream --format '{{.Name}} {{.MemUsage}} {{.CPUPerc}}' \
$($COMPOSE ps -q api) >> "$TMP/stats.txt" 2>/dev/null || true
i=$((i + 1))
sleep 0.35
done
) &
stats_pid=$!
t0=$(python3 -c 'import time; print(time.time())')
i=0
while [ "$i" -lt "$N" ]; do
(
python3 - "$BASE_URL" "$CHAT_ID" "$TOKEN" "$TMP/load.png" "$TMP/res_$i.json" <<'PY'
import json, sys, time, urllib.request
base, chat, token, path, out = sys.argv[1:6]
url = f"{base}/api/v1/chats/{chat}/attachments?filename=load.png"
data = open(path, "rb").read()
req = urllib.request.Request(
url, data=data, method="POST",
headers={"Authorization": f"Bearer {token}", "Content-Type": "image/png",
"Content-Length": str(len(data))})
t0 = time.time()
try:
with urllib.request.urlopen(req, timeout=180) as resp:
code = resp.status
body = resp.read()
except Exception as e:
code = 0
body = str(e).encode()
dt = (time.time() - t0) * 1000.0
open(out, "w").write(json.dumps({"code": code, "latency_ms": dt, "bytes": len(body)}))
PY
) &
i=$((i + 1))
done
wait
t1=$(python3 -c 'import time; print(time.time())')
kill "$stats_pid" 2>/dev/null || true
wait "$stats_pid" 2>/dev/null || true
python3 - "$TMP" "$OUT" "$N" "$SIZE" "$t0" "$t1" <<'PY'
import json, os, re, sys
tmpdir, out_path, n, size = sys.argv[1], sys.argv[2], int(sys.argv[3]), int(sys.argv[4])
wall = (float(sys.argv[6]) - float(sys.argv[5])) * 1000.0
def line_mib(line):
m = re.search(r"([\d.]+)(MiB|GiB)", line)
if not m:
return None
v = float(m.group(1))
return v * 1024.0 if m.group(2) == "GiB" else v
def sample_peaks(path):
"""One docker-stats snapshot = two api lines; return peak per snapshot."""
peaks = []
cur = 0.0
nlines = 0
if not os.path.exists(path):
return peaks
for line in open(path):
v = line_mib(line)
if v is None:
continue
cur = max(cur, v)
nlines += 1
if nlines % 2 == 0:
peaks.append(cur)
cur = 0.0
if nlines % 2 == 1 and cur > 0:
peaks.append(cur)
return peaks
def cpu_peak(path):
peak = 0.0
if not os.path.exists(path):
return peak
for line in open(path):
c = re.search(r"([\d.]+)%", line)
if c:
peak = max(peak, float(c.group(1)))
return peak
base_peaks = sample_peaks(os.path.join(tmpdir, "baseline.txt"))
load_peaks = sample_peaks(os.path.join(tmpdir, "stats.txt"))
# Baseline = max RSS observed during warm uploads (not post-trim idle floor).
base_mib = max(base_peaks) if base_peaks else 0.0
peak_mib = max(load_peaks) if load_peaks else 0.0
# If load peak is below warm peak, delta is 0 (no extra RSS under concurrency).
cpu_p = cpu_peak(os.path.join(tmpdir, "stats.txt"))
delta = max(0.0, peak_mib - base_mib)
ok = 0
lat = []
codes = {}
for i in range(n):
row = json.load(open(os.path.join(tmpdir, f"res_{i}.json")))
code = str(row["code"])
codes[code] = codes.get(code, 0) + 1
if code == "201":
ok += 1
lat.append(float(row["latency_ms"]))
def pct(xs, p):
if not xs:
return None
xs = sorted(xs)
k = min(len(xs) - 1, max(0, int(round((p / 100.0) * (len(xs) - 1)))))
return round(xs[k], 1)
linear_mib = (n * size) / (1024 * 1024)
rps = (ok / (wall / 1000.0)) if wall > 0 else None
summary = {
"concurrency": n,
"fixture_bytes": size,
"http_codes": codes,
"success": ok,
"wall_ms": round(wall, 1),
"latency_ms": {
"p50": pct(lat, 50),
"p95": pct(lat, 95),
"p99": pct(lat, 99),
"max": round(max(lat), 1) if lat else None,
},
"approx_success_rps": round(rps, 3) if rps else None,
"api_rss_baseline_mib": round(base_mib, 2),
"api_rss_peak_mib": round(peak_mib, 2),
"api_rss_delta_mib": round(delta, 2),
"api_cpu_peak_percent": round(cpu_p, 2),
"naive_full_buffer_mib": round(linear_mib, 2),
"max_concurrent_uploads": int(os.environ.get("MAX_CONCURRENT_UPLOADS", "3")),
"note": (
"RSS capped by MAX_CONCURRENT_UPLOADS (default 3) per API process; "
"slot taken before buffering body >256KiB; malloc_trim after each upload. "
"rss_peak ≤ baseline + MAX_CONCURRENT_UPLOADS × (body + decoded_raster). "
"Closed-loop client; RPS is throughput of confirmed 201s."
),
}
open(out_path, "w").write(json.dumps(summary, indent=2) + "\n")
print(json.dumps(summary, indent=2))
if ok < n:
raise SystemExit(1)
if not (delta < linear_mib):
print(
f"FAIL criterion: rss_delta={delta:.1f}MiB >= naive={linear_mib:.1f}MiB",
file=sys.stderr,
)
raise SystemExit(2)
PY
echo "load_uploads: wrote $OUT"
echo "PASS load_uploads"