Files
mtusi2026lab7/student-tests/upload_concurrency.py
T
2026-11-04 19:22:35 +03:00

152 lines
4.8 KiB
Python

#!/usr/bin/env python3
"""Run ten simultaneous near-6-MiB uploads and sample API container memory."""
from concurrent.futures import ThreadPoolExecutor
import json
import os
import re
import subprocess
import threading
import time
import uuid
from io import BytesIO
from pathlib import Path
import sys
from PIL import Image
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
from client import Client, arguments, check, run
UNIT = {"B": 1, "KiB": 1024, "MiB": 1024**2, "GiB": 1024**3}
USAGE_RE = re.compile(r"([\d.]+)(B|KiB|MiB|GiB)\s*/")
def usage_bytes(line):
match = USAGE_RE.search(line)
return int(float(match.group(1)) * UNIT[match.group(2)]) if match else 0
def api_containers():
result = subprocess.run(
["docker", "compose", "ps", "--quiet", "--status", "running", "api"],
cwd=ROOT,
check=True,
capture_output=True,
text=True,
)
ids = result.stdout.split()
check(len(ids) >= 2, "at least two running API containers are required")
return ids
def sample_once(ids):
output = subprocess.run(
["docker", "stats", "--no-stream", "--format", "{{.MemUsage}}", *ids],
check=True,
capture_output=True,
text=True,
).stdout.splitlines()
values = [usage_bytes(line) for line in output]
check(len(values) == len(ids) and all(values), "could not parse docker stats memory usage")
return max(values)
def main():
args = arguments("Bounded-memory ten-client upload experiment")
client = Client(args.base_url, args.ca_file)
alice, _, _, chat = client.bootstrap()
width = height = 1400
content = BytesIO()
Image.frombytes("RGB", (width, height), os.urandom(width * height * 3)).save(
content, format="PNG", compress_level=0
)
payload = content.getvalue()
check(0 < len(payload) <= 10 * 1024 * 1024, "synthetic PNG is outside the upload size limit")
ids = api_containers()
baseline = sample_once(ids)
samples = []
stats_process = subprocess.Popen(
["docker", "stats", "--format", "{{.MemUsage}}", *ids],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True,
bufsize=1,
)
def sampler():
if stats_process.stdout is None:
return
for line in stats_process.stdout:
value = usage_bytes(line)
if value:
samples.append(value)
sampling_thread = threading.Thread(target=sampler, daemon=True)
sampling_thread.start()
count = 10
barrier = threading.Barrier(count)
def upload(index):
worker = Client(args.base_url, args.ca_file)
barrier.wait(timeout=15)
return worker.call(
"POST",
"/api/v1/chats/" + chat["id"] + "/attachments?filename=load-" + str(index) + ".png",
202,
actor=alice,
raw=payload,
headers={"Content-Type": "image/png"},
schema="Attachment",
)
started = time.monotonic()
try:
with ThreadPoolExecutor(max_workers=count) as pool:
responses = list(pool.map(upload, range(count)))
finally:
stopped_by_test = False
if stats_process.poll() is None:
stats_process.kill()
stopped_by_test = True
try:
stats_process.wait(timeout=5)
except subprocess.TimeoutExpired:
stats_process.kill()
stats_process.wait(timeout=5)
sampling_thread.join(timeout=5)
stats_error = stats_process.stderr.read() if stats_process.stderr is not None else ""
check(
stats_process.returncode == 0 or (stopped_by_test and stats_process.returncode == -9),
"docker stats sampling process failed (exit "
+ str(stats_process.returncode)
+ "): "
+ stats_error.strip()[:300],
)
check(bool(samples), "docker stats produced no memory samples")
peak = max([baseline] + samples)
check(len(responses) == count and all(response["status"] == 202 for response in responses), "an upload failed")
print(
json.dumps(
{
"uploads": count,
"concurrency": count,
"bytes_per_upload": len(payload),
"duration_seconds": round(time.monotonic() - started, 3),
"api_containers": len(ids),
"sample_count": len(samples) // len(ids),
"baseline_max_container_bytes": baseline,
"peak_max_container_bytes": peak,
"peak_delta_bytes": max(0, peak - baseline),
"sampling_method": "continuous docker stats stream; Docker/CLI sampling cadence",
"memory_scope": "Docker-reported API container working set; LB, DB, S3, and host cache excluded",
},
sort_keys=True,
)
)
if __name__ == "__main__":
run(main)