107 lines
4.7 KiB
Python
107 lines
4.7 KiB
Python
#!/usr/bin/env python3
|
|
"""Ten concurrent bounded uploads, with API-container peak memory sampling."""
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
import json
|
|
import http.client
|
|
from pathlib import Path
|
|
import subprocess
|
|
import sys
|
|
import threading
|
|
import time
|
|
from urllib.parse import quote
|
|
from urllib.parse import urlsplit
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT / "tests"))
|
|
from client import Client, arguments, check, run
|
|
|
|
|
|
def api_memory() -> dict[str, int]:
|
|
ids = subprocess.check_output(["docker", "compose", "ps", "-q", "api-a", "api-b"], cwd=ROOT, text=True).split()
|
|
return {container[:12]: int(subprocess.check_output(
|
|
["docker", "exec", container, "cat", "/sys/fs/cgroup/memory.current"], text=True))
|
|
for container in ids}
|
|
|
|
|
|
def main():
|
|
def configure(parser):
|
|
parser.add_argument("--output", default="evidence/upload-memory.json")
|
|
parser.add_argument("--size-mib", type=int, default=5)
|
|
args = arguments(__doc__, configure)
|
|
check(1 <= args.size_mib <= 10, "--size-mib must be 1..10")
|
|
client = Client(args.base_url, args.ca_file, timeout=60)
|
|
alice, _bob, _eve, chat = client.bootstrap()
|
|
seed = (ROOT / "tests/fixtures/sample.png").read_bytes()
|
|
payload = seed + b"\0" * (args.size_mib * 1024 * 1024 - len(seed))
|
|
baseline, peak, samples, stop = api_memory(), {}, [], threading.Event()
|
|
check(len(baseline) == 2, "memory experiment requires two running APIs")
|
|
sampling_errors = []
|
|
timestamps = []
|
|
barrier = threading.Barrier(10)
|
|
|
|
def sampler():
|
|
while not stop.is_set():
|
|
try:
|
|
values = api_memory()
|
|
check(set(values) == set(baseline), "API containers changed during measurement")
|
|
except Exception as error:
|
|
sampling_errors.append(type(error).__name__)
|
|
return
|
|
timestamps.append(time.monotonic())
|
|
samples.append(values)
|
|
for name, value in values.items():
|
|
peak[name] = max(peak.get(name, 0), value)
|
|
time.sleep(.1)
|
|
|
|
def upload(index):
|
|
barrier.wait(timeout=15)
|
|
parsed = urlsplit(client.base)
|
|
connection = http.client.HTTPConnection(parsed.hostname, parsed.port, timeout=60)
|
|
|
|
def chunks():
|
|
for offset in range(0, len(payload), 512 * 1024):
|
|
yield payload[offset:offset + 512 * 1024]
|
|
# Keep all ten client streams open long enough for multiple memory samples.
|
|
time.sleep(.5)
|
|
|
|
connection.request("POST", "/api/v1/chats/" + chat["id"] + "/attachments?filename=" + quote(f"load-{index}.png"),
|
|
body=chunks(), encode_chunked=True,
|
|
headers={"Authorization": "Bearer " + alice["token"], "Content-Type": "image/png"})
|
|
response = connection.getresponse()
|
|
body = response.read()
|
|
connection.close()
|
|
check(response.status == 201, "upload %s returned %s: %s" % (index, response.status, body[:200]))
|
|
return json.loads(body)
|
|
|
|
started = time.monotonic()
|
|
thread = threading.Thread(target=sampler, daemon=True)
|
|
thread.start()
|
|
try:
|
|
with ThreadPoolExecutor(max_workers=10) as pool:
|
|
attachments = list(pool.map(upload, range(10)))
|
|
finally:
|
|
stop.set()
|
|
thread.join(timeout=5)
|
|
elapsed = time.monotonic() - started
|
|
check(len({item["id"] for item in attachments}) == 10, "uploads were not distinct")
|
|
check(not sampling_errors and len(samples) >= 2 and not thread.is_alive(),
|
|
"memory sampler failed or did not collect enough samples: " + repr(sampling_errors))
|
|
report = {
|
|
"uploads": 10, "file_bytes": len(payload), "concurrency": 10,
|
|
"api_upload_concurrency_limit": 1, "elapsed_seconds": round(elapsed, 3),
|
|
"baseline_memory_bytes": baseline, "peak_memory_bytes": peak,
|
|
"peak_growth_bytes": {name: peak.get(name, 0) - baseline.get(name, 0) for name in baseline},
|
|
"samples": len(samples),
|
|
"sample_intervals_seconds": [round(b-a, 3) for a, b in zip(timestamps, timestamps[1:])],
|
|
"method": "cgroup memory.current via Docker; 0.1s pause AFTER each poll, actual intervals include Docker latency; ten clients start at a barrier and send chunked 512 KiB blocks every 0.5s. Peak is sampled, not continuous.",
|
|
}
|
|
destination = ROOT / args.output
|
|
destination.parent.mkdir(parents=True, exist_ok=True)
|
|
destination.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n")
|
|
print(json.dumps(report, ensure_ascii=False))
|
|
check(all(value <= 512 * 1024 * 1024 for value in peak.values()), "API exceeded configured memory limit")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
run(main)
|