Files
mtusi2026lab4/student-tests/upload_memory.py
T

119 lines
5.3 KiB
Python

"""Usage (стенд запущен; на время эксперимента лучше остановить другие стенды):
python student-tests/upload_memory.py --output evidence/upload-memory.json
N одновременных загрузок PNG ~10 MiB через LB при двух настройках буфера тела запроса
(UPLOAD_MEMORY_BYTES = 1 MiB в RAM, дальше временный файл — рабочая настройка; и 16 MiB — весь файл в RAM).
Перед каждым прогоном реплики перезапускаются (memory.peak cgroup считается с момента старта контейнера);
во время прогона каждые 0,1 с снимаются memory.current и anon/file из memory.stat каждой реплики.
"""
import argparse
from concurrent.futures import ThreadPoolExecutor
import http.client
import json
import os
import subprocess
import threading
import time
import urllib.parse
from helpers import BASE_URL, api_containers, compose, new_chat, new_user, wait_in_rotation, wait_ready
from imagegen import noise_png
MIB = 1024 * 1024
SAMPLER = ('while true; do echo "current $(cat /sys/fs/cgroup/memory.current)"; '
'grep -E "^(anon|file) " /sys/fs/cgroup/memory.stat; sleep 0.1; done')
class Sampler:
def __init__(self, container):
self.container, self.peaks = container, {'current': 0, 'anon': 0, 'file': 0}
self.process = subprocess.Popen(['docker', 'exec', container, 'sh', '-c', SAMPLER],
stdout=subprocess.PIPE, text=True)
self.thread = threading.Thread(target=self.read, daemon=True)
self.thread.start()
def read(self):
for line in self.process.stdout:
name, _, value = line.partition(' ')
if name in self.peaks and value.strip().isdigit():
self.peaks[name] = max(self.peaks[name], int(value))
def stop(self):
self.process.terminate()
self.process.wait()
return {name: round(value / MIB, 1) for name, value in self.peaks.items()}
def cgroup(container, name):
return int(subprocess.run(['docker', 'exec', container, 'cat', '/sys/fs/cgroup/' + name],
capture_output=True, text=True).stdout.strip())
def post(token, chat_id, data):
target = urllib.parse.urlsplit(BASE_URL)
connection = http.client.HTTPConnection(target.hostname, target.port, timeout=120)
started = time.monotonic()
try:
connection.request('POST', '/api/v1/chats/%s/attachments?filename=big.png' % chat_id, body=data,
headers={'Authorization': 'Bearer ' + token, 'Content-Type': 'image/png'})
status = connection.getresponse().status
except OSError as error:
status = type(error).__name__
connection.close()
return status, time.monotonic() - started
def restart_api(memory_bytes):
env = dict(os.environ, UPLOAD_MEMORY_BYTES=str(memory_bytes))
compose('up', '-d', '--no-deps', '--no-build', '--force-recreate', 'api', env=env)
time.sleep(3)
wait_ready()
for container in api_containers():
wait_in_rotation(container)
def run(label, memory_bytes, parallel, data):
restart_api(memory_bytes)
alice = new_user('memory')
chat = new_chat(alice)
replicas = api_containers()
idle = {c[:12]: round(cgroup(c, 'memory.current') / MIB, 1) for c in replicas}
samplers = [Sampler(c) for c in replicas]
started = time.monotonic()
with ThreadPoolExecutor(max_workers=parallel) as pool:
results = list(pool.map(lambda _: post(alice['token'], chat['id'], data), range(parallel)))
wall = time.monotonic() - started
time.sleep(0.5)
sampled = {c[:12]: s.stop() for c, s in zip(replicas, samplers)}
peak = {c[:12]: round(cgroup(c, 'memory.peak') / MIB, 1) for c in replicas}
limit = round(cgroup(replicas[0], 'memory.max') / MIB)
statuses = [r[0] for r in results]
summary = {'label': label, 'upload_memory_bytes': memory_bytes, 'parallel': parallel,
'file_mib': round(len(data) / MIB, 2), 'ok': statuses.count(201),
'errors': sorted({str(s) for s in statuses if s != 201}), 'wall_s': round(wall, 2),
'max_request_s': round(max(r[1] for r in results), 2), 'limit_mib': limit,
'idle_current_mib': idle, 'peak_since_start_mib': peak, 'sampled_peak_mib': sampled}
worst_anon = max(v['anon'] for v in sampled.values())
print('%-28s ok=%2d/%d wall=%5.1fs peak(memory.peak)=%s anon max=%.1f MiB'
% (label, summary['ok'], parallel, wall, list(peak.values()), worst_anon))
return summary
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--output')
parser.add_argument('--parallel', default='10,20')
args = parser.parse_args()
data = noise_png(10 * MIB - 50000)
runs = [run('spool 1 MiB, %s parallel' % n, MIB, int(n), data) for n in args.parallel.split(',')]
runs.append(run('all in RAM, %s parallel' % args.parallel.split(',')[-1], 16 * MIB,
int(args.parallel.split(',')[-1]), data))
restart_api(MIB)
if args.output:
with open(args.output, 'w', encoding='utf-8') as file:
json.dump({'runs': runs}, file, ensure_ascii=False, indent=2)
if __name__ == '__main__':
main()