#!/usr/bin/env python3 """Ten concurrent bounded uploads, with API-container peak memory sampling.""" from concurrent.futures import ThreadPoolExecutor import json import http.client from pathlib import Path import subprocess import sys import threading import time from urllib.parse import quote from urllib.parse import urlsplit ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT / "tests")) from client import Client, arguments, check, run def api_memory() -> dict[str, int]: ids = subprocess.check_output(["docker", "compose", "ps", "-q", "api-a", "api-b"], cwd=ROOT, text=True).split() return {container[:12]: int(subprocess.check_output( ["docker", "exec", container, "cat", "/sys/fs/cgroup/memory.current"], text=True)) for container in ids} def main(): def configure(parser): parser.add_argument("--output", default="evidence/upload-memory.json") parser.add_argument("--size-mib", type=int, default=5) args = arguments(__doc__, configure) check(1 <= args.size_mib <= 10, "--size-mib must be 1..10") client = Client(args.base_url, args.ca_file, timeout=60) alice, _bob, _eve, chat = client.bootstrap() seed = (ROOT / "tests/fixtures/sample.png").read_bytes() payload = seed + b"\0" * (args.size_mib * 1024 * 1024 - len(seed)) baseline, peak, samples, stop = api_memory(), {}, [], threading.Event() check(len(baseline) == 2, "memory experiment requires two running APIs") sampling_errors = [] timestamps = [] barrier = threading.Barrier(10) def sampler(): while not stop.is_set(): try: values = api_memory() check(set(values) == set(baseline), "API containers changed during measurement") except Exception as error: sampling_errors.append(type(error).__name__) return timestamps.append(time.monotonic()) samples.append(values) for name, value in values.items(): peak[name] = max(peak.get(name, 0), value) time.sleep(.1) def upload(index): barrier.wait(timeout=15) parsed = urlsplit(client.base) connection = http.client.HTTPConnection(parsed.hostname, parsed.port, timeout=60) def chunks(): for offset in range(0, len(payload), 512 * 1024): yield payload[offset:offset + 512 * 1024] # Keep all ten client streams open long enough for multiple memory samples. time.sleep(.5) connection.request("POST", "/api/v1/chats/" + chat["id"] + "/attachments?filename=" + quote(f"load-{index}.png"), body=chunks(), encode_chunked=True, headers={"Authorization": "Bearer " + alice["token"], "Content-Type": "image/png"}) response = connection.getresponse() body = response.read() connection.close() check(response.status == 201, "upload %s returned %s: %s" % (index, response.status, body[:200])) return json.loads(body) started = time.monotonic() thread = threading.Thread(target=sampler, daemon=True) thread.start() try: with ThreadPoolExecutor(max_workers=10) as pool: attachments = list(pool.map(upload, range(10))) finally: stop.set() thread.join(timeout=5) elapsed = time.monotonic() - started check(len({item["id"] for item in attachments}) == 10, "uploads were not distinct") check(not sampling_errors and len(samples) >= 2 and not thread.is_alive(), "memory sampler failed or did not collect enough samples: " + repr(sampling_errors)) report = { "uploads": 10, "file_bytes": len(payload), "concurrency": 10, "api_upload_concurrency_limit": 1, "elapsed_seconds": round(elapsed, 3), "baseline_memory_bytes": baseline, "peak_memory_bytes": peak, "peak_growth_bytes": {name: peak.get(name, 0) - baseline.get(name, 0) for name in baseline}, "samples": len(samples), "sample_intervals_seconds": [round(b-a, 3) for a, b in zip(timestamps, timestamps[1:])], "method": "cgroup memory.current via Docker; 0.1s pause AFTER each poll, actual intervals include Docker latency; ten clients start at a barrier and send chunked 512 KiB blocks every 0.5s. Peak is sampled, not continuous.", } destination = ROOT / args.output destination.parent.mkdir(parents=True, exist_ok=True) destination.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n") print(json.dumps(report, ensure_ascii=False)) check(all(value <= 512 * 1024 * 1024 for value in peak.values()), "API exceeded configured memory limit") if __name__ == "__main__": run(main)