#!/usr/bin/env python3 from concurrent.futures import ThreadPoolExecutor import hashlib import json import re import subprocess import threading import time from pathlib import Path from client import Client, Failure, arguments, check, run from image_gen import png ROOT = Path(__file__).resolve().parent.parent UNITS = {'B': 1, 'KiB': 1024, 'MiB': 1024 ** 2, 'GiB': 1024 ** 3} def to_mib(text): value, unit = re.match(r'([\d.]+)\s*([A-Za-z]+)', text).groups() return float(value) * UNITS[unit] / 1024 ** 2 def sample(stop, peaks): proc = subprocess.Popen(['docker', 'stats', '--format', '{{.Name}} {{.MemUsage}}'], cwd=ROOT, stdout=subprocess.PIPE, text=True) try: for line in proc.stdout: line = re.sub(r'\x1b\[[0-9;]*[A-Za-z]', '', line).strip() m = re.match(r'(\S+-api-\d+)\s+(\S+ ?\S+) /', line) if m: peaks[m.group(1)] = max(peaks.get(m.group(1), 0), to_mib(m.group(2))) if stop.is_set(): break finally: proc.terminate() def main(): def configure(p): p.add_argument('--concurrency', type=int, default=12) p.add_argument('--rounds', type=int, default=3) p.add_argument('--output') args = arguments('Concurrent upload memory experiment', configure) check(args.concurrency >= 10, 'use at least 10 concurrent uploads') c = Client(args.base_url, args.ca_file, timeout=60) alice, bob, eve, chat = c.bootstrap() data = png(1800, 1800, noise=True) digest = hashlib.sha256(data).hexdigest() path = '/api/v1/chats/' + chat['id'] + '/attachments?filename=load.png' idle, stop = {}, threading.Event() t = threading.Thread(target=sample, args=(stop, idle)) t.start() time.sleep(4) stop.set() t.join() peaks, stop = {}, threading.Event() t = threading.Thread(target=sample, args=(stop, peaks)) t.start() time.sleep(2) barrier = threading.Barrier(args.concurrency) results = [] def worker(_): local = Client(args.base_url, args.ca_file, timeout=60) barrier.wait() out = [] for _ in range(args.rounds): try: r = local.call('POST', path, 202, actor=alice, raw=data, headers={'Content-Type': 'image/png'}, schema='Attachment') out.append(r['json']['sha256'] == digest) except Failure: out.append(False) return out started = time.monotonic() with ThreadPoolExecutor(max_workers=args.concurrency) as pool: for out in pool.map(worker, range(args.concurrency)): results.extend(out) elapsed = time.monotonic() - started time.sleep(2) stop.set() t.join() limit = 192 report = { 'file_bytes': len(data), 'concurrent_uploads': args.concurrency, 'rounds': args.rounds, 'uploads_ok': sum(results), 'uploads_failed': len(results) - sum(results), 'elapsed_seconds': round(elapsed, 2), 'api_memory_limit_mib': limit, 'idle_memory_mib': {k: round(v, 1) for k, v in sorted(idle.items())}, 'peak_memory_mib': {k: round(v, 1) for k, v in sorted(peaks.items())}, 'buffering_all_files_would_need_mib': round(args.concurrency * len(data) / 1024 ** 2, 1), } text = json.dumps(report, indent=2) if args.output: Path(args.output).write_text(text + '\n') print(text) check(report['uploads_failed'] == 0, 'some uploads failed') check(max(peaks.values()) < limit, 'peak memory reached the limit') if __name__ == '__main__': run(main)