Files
mtusi2026lab5/student_tests/concurrent_upload_memory.py
T
2026-11-19 10:57:39 +03:00

146 lines
5.2 KiB
Python

#!/usr/bin/env python3
"""Run 10+ simultaneous valid PNG uploads and sample aggregate API container RAM."""
import argparse
import binascii
import concurrent.futures
import json
import random
import re
import struct
import subprocess
import sys
import threading
import time
import zlib
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / 'tests'))
from client import Client, Failure, check, run # noqa: E402
def png_chunk(kind, payload):
return (struct.pack('>I', len(payload)) + kind + payload
+ struct.pack('>I', binascii.crc32(kind + payload) & 0xffffffff))
def make_png(width=1800, height=1200):
rng = random.Random(20261111)
rows = bytearray()
for _ in range(height):
rows.append(0)
rows.extend(rng.randbytes(width * 3))
return (b'\x89PNG\r\n\x1a\n'
+ png_chunk(b'IHDR', struct.pack('>IIBBBBB', width, height, 8, 2, 0, 0, 0))
+ png_chunk(b'IDAT', zlib.compress(bytes(rows), 1))
+ png_chunk(b'IEND', b''))
def memory_bytes(value):
match = re.fullmatch(r'([0-9]+(?:\.[0-9]+)?)\s*([A-Za-z]+)', value.strip())
if not match:
raise ValueError('unrecognized Docker memory value: ' + value)
number, unit = match.groups()
factors = {'B': 1, 'kB': 1000, 'KiB': 1024, 'MB': 1000**2,
'MiB': 1024**2, 'GB': 1000**3, 'GiB': 1024**3}
return int(float(number) * factors[unit])
def api_containers(env_file):
result = subprocess.run(
['docker', 'compose', '--env-file', env_file, 'ps', '-q', 'api'],
cwd=ROOT, check=True, capture_output=True, text=True)
ids = [line for line in result.stdout.splitlines() if line]
check(len(ids) >= 2, 'memory experiment requires at least two API containers')
return ids
def sample_total_memory(container_ids):
result = subprocess.run(
['docker', 'stats', '--no-stream', '--format', '{{.MemUsage}}', *container_ids],
check=True, capture_output=True, text=True)
usages = []
for line in result.stdout.splitlines():
if line.strip():
usages.append(memory_bytes(line.split('/', 1)[0].strip()))
check(len(usages) == len(container_ids), 'docker stats did not return every API container')
return sum(usages)
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--base-url', default='http://localhost:8080')
parser.add_argument('--compose-env-file', default='.env')
parser.add_argument('--concurrency', type=int, default=12)
parser.add_argument('--output', default='evidence/upload-memory.json')
args = parser.parse_args()
check(args.concurrency >= 10, 'experiment requires concurrency >= 10')
bootstrap = Client(args.base_url, timeout=30)
alice, bob, _, chat = bootstrap.bootstrap()
payload = make_png()
check(len(payload) <= 10_485_760, 'generated PNG exceeds upload limit')
containers = api_containers(args.compose_env_file)
baseline = sample_total_memory(containers)
peak = baseline
samples = []
stop = threading.Event()
def monitor():
nonlocal peak
while not stop.is_set():
try:
current = sample_total_memory(containers)
samples.append(current)
peak = max(peak, current)
except (subprocess.SubprocessError, Failure, ValueError):
pass
stop.wait(0.1)
barrier = threading.Barrier(args.concurrency)
def upload(index):
client = Client(args.base_url, timeout=30)
barrier.wait(timeout=30)
path = (f"/api/v1/chats/{chat['id']}/attachments"
f"?filename=memory-{index}.png")
return client.call('POST', path, 202, actor=alice, raw=payload,
headers={'Content-Type': 'image/png'}, schema='Attachment')['json']['id']
monitor_thread = threading.Thread(target=monitor, daemon=True)
monitor_thread.start()
started = time.monotonic()
try:
with concurrent.futures.ThreadPoolExecutor(max_workers=args.concurrency) as pool:
ids = list(pool.map(upload, range(args.concurrency)))
finally:
stop.set()
monitor_thread.join(timeout=5)
duration = time.monotonic() - started
check(len(ids) == args.concurrency and len(set(ids)) == len(ids),
'not every concurrent upload completed exactly once')
result = {
'concurrency': args.concurrency,
'successful_uploads': len(ids),
'payload_bytes_each': len(payload),
'duration_seconds': round(duration, 3),
'api_replicas': len(containers),
'memory_samples': len(samples),
'baseline_api_memory_bytes': baseline,
'peak_api_memory_bytes': peak,
'peak_increase_bytes': max(0, peak - baseline),
'configured_max_concurrent_uploads_per_replica': 3,
'configured_max_upload_bytes': 10_485_760,
'configured_max_image_pixels': 40_000_000,
}
output = ROOT / args.output
output.parent.mkdir(parents=True, exist_ok=True)
output.write_text(json.dumps(result, ensure_ascii=False, indent=2) + '\n', encoding='utf-8')
print(json.dumps(result, ensure_ascii=False))
print('PASS: concurrent upload memory experiment')
if __name__ == '__main__':
run(main)