Files
mtusi2026lab4-fork/tests/upload_memory.py

107 lines
3.6 KiB
Python

#!/usr/bin/env python3
from concurrent.futures import ThreadPoolExecutor
import hashlib
import json
import re
import subprocess
import threading
import time
from pathlib import Path
from client import Client, Failure, arguments, check, run
from image_gen import png
ROOT = Path(__file__).resolve().parent.parent
UNITS = {'B': 1, 'KiB': 1024, 'MiB': 1024 ** 2, 'GiB': 1024 ** 3}
def to_mib(text):
value, unit = re.match(r'([\d.]+)\s*([A-Za-z]+)', text).groups()
return float(value) * UNITS[unit] / 1024 ** 2
def sample(stop, peaks):
proc = subprocess.Popen(['docker', 'stats', '--format', '{{.Name}} {{.MemUsage}}'], cwd=ROOT,
stdout=subprocess.PIPE, text=True)
try:
for line in proc.stdout:
line = re.sub(r'\x1b\[[0-9;]*[A-Za-z]', '', line).strip()
m = re.match(r'(\S+-api-\d+)\s+(\S+ ?\S+) /', line)
if m:
peaks[m.group(1)] = max(peaks.get(m.group(1), 0), to_mib(m.group(2)))
if stop.is_set():
break
finally:
proc.terminate()
def main():
def configure(p):
p.add_argument('--concurrency', type=int, default=12)
p.add_argument('--rounds', type=int, default=3)
p.add_argument('--output')
args = arguments('Concurrent upload memory experiment', configure)
check(args.concurrency >= 10, 'use at least 10 concurrent uploads')
c = Client(args.base_url, args.ca_file, timeout=60)
alice, bob, eve, chat = c.bootstrap()
data = png(1800, 1800, noise=True)
digest = hashlib.sha256(data).hexdigest()
path = '/api/v1/chats/' + chat['id'] + '/attachments?filename=load.png'
idle, stop = {}, threading.Event()
t = threading.Thread(target=sample, args=(stop, idle))
t.start()
time.sleep(4)
stop.set()
t.join()
peaks, stop = {}, threading.Event()
t = threading.Thread(target=sample, args=(stop, peaks))
t.start()
time.sleep(2)
barrier = threading.Barrier(args.concurrency)
results = []
def worker(_):
local = Client(args.base_url, args.ca_file, timeout=60)
barrier.wait()
out = []
for _ in range(args.rounds):
try:
r = local.call('POST', path, 201, actor=alice, raw=data, headers={'Content-Type': 'image/png'}, schema='Attachment')
out.append(r['json']['sha256'] == digest)
except Failure:
out.append(False)
return out
started = time.monotonic()
with ThreadPoolExecutor(max_workers=args.concurrency) as pool:
for out in pool.map(worker, range(args.concurrency)):
results.extend(out)
elapsed = time.monotonic() - started
time.sleep(2)
stop.set()
t.join()
limit = 192
report = {
'file_bytes': len(data),
'concurrent_uploads': args.concurrency,
'rounds': args.rounds,
'uploads_ok': sum(results),
'uploads_failed': len(results) - sum(results),
'elapsed_seconds': round(elapsed, 2),
'api_memory_limit_mib': limit,
'idle_memory_mib': {k: round(v, 1) for k, v in sorted(idle.items())},
'peak_memory_mib': {k: round(v, 1) for k, v in sorted(peaks.items())},
'buffering_all_files_would_need_mib': round(args.concurrency * len(data) / 1024 ** 2, 1),
}
text = json.dumps(report, indent=2)
if args.output:
Path(args.output).write_text(text + '\n')
print(text)
check(report['uploads_failed'] == 0, 'some uploads failed')
check(max(peaks.values()) < limit, 'peak memory reached the limit')
if __name__ == '__main__':
run(main)