146 lines
5.2 KiB
Python
146 lines
5.2 KiB
Python
#!/usr/bin/env python3
|
|
"""Run 10+ simultaneous valid PNG uploads and sample aggregate API container RAM."""
|
|
import argparse
|
|
import binascii
|
|
import concurrent.futures
|
|
import json
|
|
import random
|
|
import re
|
|
import struct
|
|
import subprocess
|
|
import sys
|
|
import threading
|
|
import time
|
|
import zlib
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT / 'tests'))
|
|
from client import Client, Failure, check, run # noqa: E402
|
|
|
|
|
|
def png_chunk(kind, payload):
|
|
return (struct.pack('>I', len(payload)) + kind + payload
|
|
+ struct.pack('>I', binascii.crc32(kind + payload) & 0xffffffff))
|
|
|
|
|
|
def make_png(width=1800, height=1200):
|
|
rng = random.Random(20261111)
|
|
rows = bytearray()
|
|
for _ in range(height):
|
|
rows.append(0)
|
|
rows.extend(rng.randbytes(width * 3))
|
|
return (b'\x89PNG\r\n\x1a\n'
|
|
+ png_chunk(b'IHDR', struct.pack('>IIBBBBB', width, height, 8, 2, 0, 0, 0))
|
|
+ png_chunk(b'IDAT', zlib.compress(bytes(rows), 1))
|
|
+ png_chunk(b'IEND', b''))
|
|
|
|
|
|
def memory_bytes(value):
|
|
match = re.fullmatch(r'([0-9]+(?:\.[0-9]+)?)\s*([A-Za-z]+)', value.strip())
|
|
if not match:
|
|
raise ValueError('unrecognized Docker memory value: ' + value)
|
|
number, unit = match.groups()
|
|
factors = {'B': 1, 'kB': 1000, 'KiB': 1024, 'MB': 1000**2,
|
|
'MiB': 1024**2, 'GB': 1000**3, 'GiB': 1024**3}
|
|
return int(float(number) * factors[unit])
|
|
|
|
|
|
def api_containers(env_file):
|
|
result = subprocess.run(
|
|
['docker', 'compose', '--env-file', env_file, 'ps', '-q', 'api'],
|
|
cwd=ROOT, check=True, capture_output=True, text=True)
|
|
ids = [line for line in result.stdout.splitlines() if line]
|
|
check(len(ids) >= 2, 'memory experiment requires at least two API containers')
|
|
return ids
|
|
|
|
|
|
def sample_total_memory(container_ids):
|
|
result = subprocess.run(
|
|
['docker', 'stats', '--no-stream', '--format', '{{.MemUsage}}', *container_ids],
|
|
check=True, capture_output=True, text=True)
|
|
usages = []
|
|
for line in result.stdout.splitlines():
|
|
if line.strip():
|
|
usages.append(memory_bytes(line.split('/', 1)[0].strip()))
|
|
check(len(usages) == len(container_ids), 'docker stats did not return every API container')
|
|
return sum(usages)
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument('--base-url', default='http://localhost:8080')
|
|
parser.add_argument('--compose-env-file', default='.env')
|
|
parser.add_argument('--concurrency', type=int, default=12)
|
|
parser.add_argument('--output', default='evidence/upload-memory.json')
|
|
args = parser.parse_args()
|
|
check(args.concurrency >= 10, 'experiment requires concurrency >= 10')
|
|
|
|
bootstrap = Client(args.base_url, timeout=30)
|
|
alice, bob, _, chat = bootstrap.bootstrap()
|
|
payload = make_png()
|
|
check(len(payload) <= 10_485_760, 'generated PNG exceeds upload limit')
|
|
containers = api_containers(args.compose_env_file)
|
|
baseline = sample_total_memory(containers)
|
|
peak = baseline
|
|
samples = []
|
|
stop = threading.Event()
|
|
|
|
def monitor():
|
|
nonlocal peak
|
|
while not stop.is_set():
|
|
try:
|
|
current = sample_total_memory(containers)
|
|
samples.append(current)
|
|
peak = max(peak, current)
|
|
except (subprocess.SubprocessError, Failure, ValueError):
|
|
pass
|
|
stop.wait(0.1)
|
|
|
|
barrier = threading.Barrier(args.concurrency)
|
|
|
|
def upload(index):
|
|
client = Client(args.base_url, timeout=30)
|
|
barrier.wait(timeout=30)
|
|
path = (f"/api/v1/chats/{chat['id']}/attachments"
|
|
f"?filename=memory-{index}.png")
|
|
return client.call('POST', path, 201, actor=alice, raw=payload,
|
|
headers={'Content-Type': 'image/png'}, schema='Attachment')['json']['id']
|
|
|
|
monitor_thread = threading.Thread(target=monitor, daemon=True)
|
|
monitor_thread.start()
|
|
started = time.monotonic()
|
|
try:
|
|
with concurrent.futures.ThreadPoolExecutor(max_workers=args.concurrency) as pool:
|
|
ids = list(pool.map(upload, range(args.concurrency)))
|
|
finally:
|
|
stop.set()
|
|
monitor_thread.join(timeout=5)
|
|
duration = time.monotonic() - started
|
|
check(len(ids) == args.concurrency and len(set(ids)) == len(ids),
|
|
'not every concurrent upload completed exactly once')
|
|
|
|
result = {
|
|
'concurrency': args.concurrency,
|
|
'successful_uploads': len(ids),
|
|
'payload_bytes_each': len(payload),
|
|
'duration_seconds': round(duration, 3),
|
|
'api_replicas': len(containers),
|
|
'memory_samples': len(samples),
|
|
'baseline_api_memory_bytes': baseline,
|
|
'peak_api_memory_bytes': peak,
|
|
'peak_increase_bytes': max(0, peak - baseline),
|
|
'configured_max_concurrent_uploads_per_replica': 3,
|
|
'configured_max_upload_bytes': 10_485_760,
|
|
'configured_max_image_pixels': 40_000_000,
|
|
}
|
|
output = ROOT / args.output
|
|
output.parent.mkdir(parents=True, exist_ok=True)
|
|
output.write_text(json.dumps(result, ensure_ascii=False, indent=2) + '\n', encoding='utf-8')
|
|
print(json.dumps(result, ensure_ascii=False))
|
|
print('PASS: concurrent upload memory experiment')
|
|
|
|
|
|
if __name__ == '__main__':
|
|
run(main)
|