forked from mtusi2026/mtusi2026lab7
152 lines
4.8 KiB
Python
152 lines
4.8 KiB
Python
#!/usr/bin/env python3
|
|
"""Run ten simultaneous near-6-MiB uploads and sample API container memory."""
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
import json
|
|
import os
|
|
import re
|
|
import subprocess
|
|
import threading
|
|
import time
|
|
import uuid
|
|
from io import BytesIO
|
|
from pathlib import Path
|
|
import sys
|
|
|
|
from PIL import Image
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT / "tests"))
|
|
from client import Client, arguments, check, run
|
|
|
|
|
|
UNIT = {"B": 1, "KiB": 1024, "MiB": 1024**2, "GiB": 1024**3}
|
|
USAGE_RE = re.compile(r"([\d.]+)(B|KiB|MiB|GiB)\s*/")
|
|
|
|
|
|
def usage_bytes(line):
|
|
match = USAGE_RE.search(line)
|
|
return int(float(match.group(1)) * UNIT[match.group(2)]) if match else 0
|
|
|
|
|
|
def api_containers():
|
|
result = subprocess.run(
|
|
["docker", "compose", "ps", "--quiet", "--status", "running", "api"],
|
|
cwd=ROOT,
|
|
check=True,
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
ids = result.stdout.split()
|
|
check(len(ids) >= 2, "at least two running API containers are required")
|
|
return ids
|
|
|
|
|
|
def sample_once(ids):
|
|
output = subprocess.run(
|
|
["docker", "stats", "--no-stream", "--format", "{{.MemUsage}}", *ids],
|
|
check=True,
|
|
capture_output=True,
|
|
text=True,
|
|
).stdout.splitlines()
|
|
values = [usage_bytes(line) for line in output]
|
|
check(len(values) == len(ids) and all(values), "could not parse docker stats memory usage")
|
|
return max(values)
|
|
|
|
|
|
def main():
|
|
args = arguments("Bounded-memory ten-client upload experiment")
|
|
client = Client(args.base_url, args.ca_file)
|
|
alice, _, _, chat = client.bootstrap()
|
|
width = height = 1400
|
|
content = BytesIO()
|
|
Image.frombytes("RGB", (width, height), os.urandom(width * height * 3)).save(
|
|
content, format="PNG", compress_level=0
|
|
)
|
|
payload = content.getvalue()
|
|
check(0 < len(payload) <= 10 * 1024 * 1024, "synthetic PNG is outside the upload size limit")
|
|
ids = api_containers()
|
|
baseline = sample_once(ids)
|
|
samples = []
|
|
stats_process = subprocess.Popen(
|
|
["docker", "stats", "--format", "{{.MemUsage}}", *ids],
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE,
|
|
text=True,
|
|
bufsize=1,
|
|
)
|
|
|
|
def sampler():
|
|
if stats_process.stdout is None:
|
|
return
|
|
for line in stats_process.stdout:
|
|
value = usage_bytes(line)
|
|
if value:
|
|
samples.append(value)
|
|
|
|
sampling_thread = threading.Thread(target=sampler, daemon=True)
|
|
sampling_thread.start()
|
|
count = 10
|
|
barrier = threading.Barrier(count)
|
|
|
|
def upload(index):
|
|
worker = Client(args.base_url, args.ca_file)
|
|
barrier.wait(timeout=15)
|
|
return worker.call(
|
|
"POST",
|
|
"/api/v1/chats/" + chat["id"] + "/attachments?filename=load-" + str(index) + ".png",
|
|
202,
|
|
actor=alice,
|
|
raw=payload,
|
|
headers={"Content-Type": "image/png"},
|
|
schema="Attachment",
|
|
)
|
|
|
|
started = time.monotonic()
|
|
try:
|
|
with ThreadPoolExecutor(max_workers=count) as pool:
|
|
responses = list(pool.map(upload, range(count)))
|
|
finally:
|
|
stopped_by_test = False
|
|
if stats_process.poll() is None:
|
|
stats_process.kill()
|
|
stopped_by_test = True
|
|
try:
|
|
stats_process.wait(timeout=5)
|
|
except subprocess.TimeoutExpired:
|
|
stats_process.kill()
|
|
stats_process.wait(timeout=5)
|
|
sampling_thread.join(timeout=5)
|
|
stats_error = stats_process.stderr.read() if stats_process.stderr is not None else ""
|
|
check(
|
|
stats_process.returncode == 0 or (stopped_by_test and stats_process.returncode == -9),
|
|
"docker stats sampling process failed (exit "
|
|
+ str(stats_process.returncode)
|
|
+ "): "
|
|
+ stats_error.strip()[:300],
|
|
)
|
|
check(bool(samples), "docker stats produced no memory samples")
|
|
peak = max([baseline] + samples)
|
|
check(len(responses) == count and all(response["status"] == 202 for response in responses), "an upload failed")
|
|
print(
|
|
json.dumps(
|
|
{
|
|
"uploads": count,
|
|
"concurrency": count,
|
|
"bytes_per_upload": len(payload),
|
|
"duration_seconds": round(time.monotonic() - started, 3),
|
|
"api_containers": len(ids),
|
|
"sample_count": len(samples) // len(ids),
|
|
"baseline_max_container_bytes": baseline,
|
|
"peak_max_container_bytes": peak,
|
|
"peak_delta_bytes": max(0, peak - baseline),
|
|
"sampling_method": "continuous docker stats stream; Docker/CLI sampling cadence",
|
|
"memory_scope": "Docker-reported API container working set; LB, DB, S3, and host cache excluded",
|
|
},
|
|
sort_keys=True,
|
|
)
|
|
)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
run(main)
|