forked from mtusi2026/mtusi2026lab5
107 lines
3.6 KiB
Python
107 lines
3.6 KiB
Python
#!/usr/bin/env python3
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
import hashlib
|
|
import json
|
|
import re
|
|
import subprocess
|
|
import threading
|
|
import time
|
|
from pathlib import Path
|
|
from client import Client, Failure, arguments, check, run
|
|
from image_gen import png
|
|
|
|
ROOT = Path(__file__).resolve().parent.parent
|
|
UNITS = {'B': 1, 'KiB': 1024, 'MiB': 1024 ** 2, 'GiB': 1024 ** 3}
|
|
|
|
|
|
def to_mib(text):
|
|
value, unit = re.match(r'([\d.]+)\s*([A-Za-z]+)', text).groups()
|
|
return float(value) * UNITS[unit] / 1024 ** 2
|
|
|
|
|
|
def sample(stop, peaks):
|
|
proc = subprocess.Popen(['docker', 'stats', '--format', '{{.Name}} {{.MemUsage}}'], cwd=ROOT,
|
|
stdout=subprocess.PIPE, text=True)
|
|
try:
|
|
for line in proc.stdout:
|
|
line = re.sub(r'\x1b\[[0-9;]*[A-Za-z]', '', line).strip()
|
|
m = re.match(r'(\S+-api-\d+)\s+(\S+ ?\S+) /', line)
|
|
if m:
|
|
peaks[m.group(1)] = max(peaks.get(m.group(1), 0), to_mib(m.group(2)))
|
|
if stop.is_set():
|
|
break
|
|
finally:
|
|
proc.terminate()
|
|
|
|
|
|
def main():
|
|
def configure(p):
|
|
p.add_argument('--concurrency', type=int, default=12)
|
|
p.add_argument('--rounds', type=int, default=3)
|
|
p.add_argument('--output')
|
|
args = arguments('Concurrent upload memory experiment', configure)
|
|
check(args.concurrency >= 10, 'use at least 10 concurrent uploads')
|
|
c = Client(args.base_url, args.ca_file, timeout=60)
|
|
alice, bob, eve, chat = c.bootstrap()
|
|
data = png(1800, 1800, noise=True)
|
|
digest = hashlib.sha256(data).hexdigest()
|
|
path = '/api/v1/chats/' + chat['id'] + '/attachments?filename=load.png'
|
|
|
|
idle, stop = {}, threading.Event()
|
|
t = threading.Thread(target=sample, args=(stop, idle))
|
|
t.start()
|
|
time.sleep(4)
|
|
stop.set()
|
|
t.join()
|
|
|
|
peaks, stop = {}, threading.Event()
|
|
t = threading.Thread(target=sample, args=(stop, peaks))
|
|
t.start()
|
|
time.sleep(2)
|
|
barrier = threading.Barrier(args.concurrency)
|
|
results = []
|
|
|
|
def worker(_):
|
|
local = Client(args.base_url, args.ca_file, timeout=60)
|
|
barrier.wait()
|
|
out = []
|
|
for _ in range(args.rounds):
|
|
try:
|
|
r = local.call('POST', path, 202, actor=alice, raw=data, headers={'Content-Type': 'image/png'}, schema='Attachment')
|
|
out.append(r['json']['sha256'] == digest)
|
|
except Failure:
|
|
out.append(False)
|
|
return out
|
|
|
|
started = time.monotonic()
|
|
with ThreadPoolExecutor(max_workers=args.concurrency) as pool:
|
|
for out in pool.map(worker, range(args.concurrency)):
|
|
results.extend(out)
|
|
elapsed = time.monotonic() - started
|
|
time.sleep(2)
|
|
stop.set()
|
|
t.join()
|
|
limit = 192
|
|
report = {
|
|
'file_bytes': len(data),
|
|
'concurrent_uploads': args.concurrency,
|
|
'rounds': args.rounds,
|
|
'uploads_ok': sum(results),
|
|
'uploads_failed': len(results) - sum(results),
|
|
'elapsed_seconds': round(elapsed, 2),
|
|
'api_memory_limit_mib': limit,
|
|
'idle_memory_mib': {k: round(v, 1) for k, v in sorted(idle.items())},
|
|
'peak_memory_mib': {k: round(v, 1) for k, v in sorted(peaks.items())},
|
|
'buffering_all_files_would_need_mib': round(args.concurrency * len(data) / 1024 ** 2, 1),
|
|
}
|
|
text = json.dumps(report, indent=2)
|
|
if args.output:
|
|
Path(args.output).write_text(text + '\n')
|
|
print(text)
|
|
check(report['uploads_failed'] == 0, 'some uploads failed')
|
|
check(max(peaks.values()) < limit, 'peak memory reached the limit')
|
|
|
|
|
|
if __name__ == '__main__':
|
|
run(main)
|