Files
mtusi2026lab4/scripts/benchmark_concurrent_uploads.py
T

305 lines
9.6 KiB
Python

import csv
import hashlib
import json
import subprocess
import sys
import threading
import time
import urllib.request
from concurrent.futures import ThreadPoolExecutor, as_completed
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
from client import Client, check
BASE_URL = "http://localhost:8080"
FIXTURE = ROOT / "tests" / "fixtures" / "sample_large.png"
API_CONTAINERS = ("mtusi-lab4-api-1", "mtusi-lab4-api-2")
CONCURRENCY = 10
OUTPUT = ROOT / "evidence" / "concurrent-upload-results.csv"
def parse_mib(value):
units = {
"B": 1 / (1024 * 1024),
"KiB": 1 / 1024,
"MiB": 1,
"GiB": 1024,
"Ki": 1 / 1024,
"Mi": 1,
"Gi": 1024,
"kB": 1000 / (1024 * 1024),
"MB": 1000 * 1000 / (1024 * 1024),
"GB": 1000 * 1000 * 1000 / (1024 * 1024),
}
for unit in sorted(units, key=len, reverse=True):
if value.endswith(unit):
return float(value[:-len(unit)].strip()) * units[unit]
raise ValueError(f"Unknown Docker memory unit: {value}")
def docker_memory():
result = subprocess.run(
[
"docker", "stats", "--no-stream",
"--format", "{{.Name}}|{{.MemUsage}}",
],
capture_output=True,
text=True,
check=True,
)
values = {}
for line in result.stdout.splitlines():
if "|" not in line:
continue
name, usage = line.split("|", 1)
if name in API_CONTAINERS:
values[name] = parse_mib(usage.split("/", 1)[0].strip())
missing = set(API_CONTAINERS) - set(values)
if missing:
raise RuntimeError("Missing API containers: " + ", ".join(missing))
return values
class MemoryMonitor:
def __init__(self):
self.running = False
self.samples = []
self.thread = None
self.errors = []
def sample(self):
while self.running:
try:
values = docker_memory()
self.samples.append(values)
except Exception as exc:
self.errors.append(type(exc).__name__)
time.sleep(0.1)
def __enter__(self):
self.baseline = docker_memory()
self.samples = [dict(self.baseline)]
self.running = True
self.thread = threading.Thread(target=self.sample, daemon=True)
self.thread.start()
return self
def __exit__(self, exc_type, exc_value, traceback):
self.running = False
self.thread.join(timeout=5)
try:
self.samples.append(docker_memory())
except Exception as exc:
self.errors.append(type(exc).__name__)
def summary(self):
peak_by_container = {
name: max(sample[name] for sample in self.samples)
for name in API_CONTAINERS
}
total_samples = [
sum(sample[name] for name in API_CONTAINERS)
for sample in self.samples
]
baseline_total = sum(self.baseline.values())
peak_total = max(total_samples)
return {
"baseline_total_mib": round(baseline_total, 2),
"peak_total_mib": round(peak_total, 2),
"peak_delta_total_mib": round(peak_total - baseline_total, 2),
"api1_peak_mib": round(peak_by_container[API_CONTAINERS[0]], 2),
"api2_peak_mib": round(peak_by_container[API_CONTAINERS[1]], 2),
"samples": len(self.samples),
"sampling_errors": len(self.errors),
}
def response_json(response):
return json.loads(response["bytes"].decode("utf-8"))
def upload_one(index, method, actor, chat, content, digest, barrier):
client = Client(BASE_URL)
barrier.wait(timeout=30)
started = time.monotonic()
if method == "proxy":
response = client.call(
"POST",
f"/api/v1/chats/{chat['id']}/attachments"
f"?filename=parallel-{index}.png",
201,
actor=actor,
raw=content,
headers={"Content-Type": "image/png"},
)
attachment = response_json(response)
api_elapsed = response["elapsed"]
api_body_bytes = len(content)
s3_body_bytes = 0
else:
body = {
"filename": f"parallel-{index}.png",
"content_type": "image/png",
"size_bytes": len(content),
"sha256": digest,
}
create_response = client.call(
"POST",
f"/api/v1/chats/{chat['id']}/attachments/presigned",
201,
actor=actor,
body=body,
)
created = response_json(create_response)
request = urllib.request.Request(
created["upload_url"],
data=content,
headers=created["headers"],
method="PUT",
)
with urllib.request.urlopen(request, timeout=90) as response:
check(response.status in (200, 201, 204), "S3 PUT failed")
complete_response = client.call(
"POST",
f"/api/v1/attachments/uploads/{created['upload_id']}/complete",
200,
actor=actor,
)
attachment = response_json(complete_response)
api_elapsed = create_response["elapsed"] + complete_response["elapsed"]
api_body_bytes = len(json.dumps(body).encode("utf-8"))
s3_body_bytes = len(content)
elapsed = time.monotonic() - started
check(attachment["status"] == "ready", "attachment is not ready")
check(attachment["size_bytes"] == len(content), "wrong attachment size")
check(attachment["sha256"] == digest, "wrong attachment checksum")
return {
"worker": index,
"elapsed_total_s": round(elapsed, 4),
"elapsed_api_s": round(api_elapsed, 4),
"client_to_api_body_bytes": api_body_bytes,
"client_to_s3_upload_bytes": s3_body_bytes,
"attachment_id": attachment["id"],
}
def run_method(method, actor, chat, content, digest):
print(f"\nStarting {CONCURRENCY} simultaneous {method} uploads...")
barrier = threading.Barrier(CONCURRENCY)
results = []
errors = []
with MemoryMonitor() as memory:
started = time.monotonic()
with ThreadPoolExecutor(max_workers=CONCURRENCY) as executor:
futures = [
executor.submit(
upload_one, index, method, actor, chat,
content, digest, barrier,
)
for index in range(1, CONCURRENCY + 1)
]
for future in as_completed(futures):
try:
results.append(future.result())
except Exception as exc:
errors.append(f"{type(exc).__name__}: {exc}")
wall_time = time.monotonic() - started
mem = memory.summary()
successful = len(results)
print(
f"{method}: success={successful}/{CONCURRENCY}, "
f"errors={len(errors)}, wall={wall_time:.3f}s, "
f"peak API RAM={mem['peak_total_mib']} MiB, "
f"delta={mem['peak_delta_total_mib']} MiB, "
f"samples={mem['samples']}"
)
for error in errors:
print("ERROR:", error)
return {
"method": method,
"concurrency": CONCURRENCY,
"file_size_bytes": len(content),
"successful_uploads": successful,
"failed_uploads": len(errors),
"wall_time_s": round(wall_time, 4),
"throughput_uploads_per_s": round(successful / wall_time, 3),
"peak_api_total_mib": mem["peak_total_mib"],
"baseline_api_total_mib": mem["baseline_total_mib"],
"peak_delta_api_total_mib": mem["peak_delta_total_mib"],
"api1_peak_mib": mem["api1_peak_mib"],
"api2_peak_mib": mem["api2_peak_mib"],
"memory_samples": mem["samples"],
"memory_sampling_errors": mem["sampling_errors"],
"avg_upload_time_s": round(
sum(item["elapsed_total_s"] for item in results) / successful, 4
) if successful else None,
"max_upload_time_s": max(
(item["elapsed_total_s"] for item in results), default=None
),
"api_body_bytes_per_upload": (
results[0]["client_to_api_body_bytes"] if results else None
),
"s3_body_bytes_per_upload": (
results[0]["client_to_s3_upload_bytes"] if results else None
),
}
def main():
if not FIXTURE.exists():
raise FileNotFoundError(FIXTURE)
content = FIXTURE.read_bytes()
if len(content) > 10 * 1024 * 1024:
raise RuntimeError("Fixture exceeds 10 MiB")
digest = hashlib.sha256(content).hexdigest()
with urllib.request.urlopen(BASE_URL + "/health/ready", timeout=10) as response:
check(response.status == 200, "API is not ready")
# Prepare identities and chat before measuring concurrent uploads.
setup = Client(BASE_URL)
alice, _, _, chat = setup.bootstrap()
rows = []
for method in ("proxy", "presigned"):
rows.append(run_method(method, alice, chat, content, digest))
OUTPUT.parent.mkdir(parents=True, exist_ok=True)
with OUTPUT.open("w", newline="", encoding="utf-8-sig") as file:
writer = csv.DictWriter(file, fieldnames=list(rows[0].keys()))
writer.writeheader()
writer.writerows(rows)
print("\nResults:", OUTPUT)
print("Fixture:", FIXTURE.name)
print("File size:", len(content), "bytes")
print("SHA-256:", digest)
if any(row["successful_uploads"] != CONCURRENCY for row in rows):
raise SystemExit("FAIL: at least one concurrent upload failed")
print("PASS: 10 concurrent uploads completed for both methods")
if __name__ == "__main__":
main()