Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
108 lines
5.4 KiB
Python
108 lines
5.4 KiB
Python
#!/usr/bin/env python3
|
||
"""Эксперимент уровня 4: N одновременных загрузок ~10 MiB и пик памяти контейнеров.
|
||
|
||
Память читается из cgroup v2 внутри контейнеров (memory.current и anon из memory.stat) каждые 0.2 с
|
||
через docker compose exec, пока идут загрузки. anon — память процессов (heap JVM и native),
|
||
memory.current дополнительно включает page cache временных файлов.
|
||
Пример: python3 tests/upload_memory.py --runs 1,10,20 --output evidence/upload_memory.json
|
||
"""
|
||
import json
|
||
import os
|
||
import struct
|
||
import subprocess
|
||
import threading
|
||
import time
|
||
import zlib
|
||
from concurrent.futures import ThreadPoolExecutor
|
||
from pathlib import Path
|
||
from client import ROOT, Client, Failure, arguments, check, run
|
||
|
||
SERVICES = ('api-1', 'api-2', 'nginx', 's3')
|
||
# Свой дедлайн: docker exec не завершает процесс в контейнере, когда клиент отключается.
|
||
SAMPLER = ('end=$(($(date +%s)+180)); while [ $(date +%s) -lt $end ]; do '
|
||
'echo $(cat /sys/fs/cgroup/memory.current) $(grep "^anon " /sys/fs/cgroup/memory.stat | cut -d" " -f2) || exit; sleep 0.2; done')
|
||
|
||
|
||
def big_png():
|
||
# 1800×1900 RGB из случайных байтов не сжимается: файл ≈9.8 MiB, чуть меньше лимита 10 MiB.
|
||
def chunk(kind, data):
|
||
return struct.pack('>I', len(data)) + kind + data + struct.pack('>I', zlib.crc32(kind + data))
|
||
raw = b''.join(b'\x00' + os.urandom(1800 * 3) for _ in range(1900))
|
||
return (b'\x89PNG\r\n\x1a\n' + chunk(b'IHDR', struct.pack('>IIBBBBB', 1800, 1900, 8, 2, 0, 0, 0))
|
||
+ chunk(b'IDAT', zlib.compress(raw, 1)) + chunk(b'IEND', b''))
|
||
|
||
|
||
class Sampler:
|
||
"""Фоновое чтение памяти одного контейнера; хранит максимум за время измерения."""
|
||
def __init__(self, service):
|
||
self.service, self.peak_current, self.peak_anon, self.base = service, 0, 0, None
|
||
self.proc = subprocess.Popen(['docker', 'compose', 'exec', '-T', service, 'sh', '-c', SAMPLER], cwd=ROOT,
|
||
stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True)
|
||
threading.Thread(target=self.read, daemon=True).start()
|
||
|
||
def read(self):
|
||
for line in self.proc.stdout:
|
||
parts = line.split()
|
||
if len(parts) != 2:
|
||
continue
|
||
current, anon = map(int, parts)
|
||
if self.base is None:
|
||
self.base = (current, anon)
|
||
self.peak_current, self.peak_anon = max(self.peak_current, current), max(self.peak_anon, anon)
|
||
|
||
def stop(self):
|
||
self.proc.terminate()
|
||
mib = lambda v: round(v / 2**20, 1)
|
||
return {'baseline_current_mib': mib(self.base[0]), 'baseline_anon_mib': mib(self.base[1]),
|
||
'peak_current_mib': mib(self.peak_current), 'peak_anon_mib': mib(self.peak_anon)}
|
||
|
||
|
||
def main():
|
||
def configure(p):
|
||
p.add_argument('--runs', default='1,10,20', help='comma-separated concurrency levels')
|
||
p.add_argument('--output', default=None)
|
||
args = arguments('Concurrent ~10 MiB uploads with peak container memory', configure)
|
||
c = Client(args.base_url, args.ca_file, timeout=120)
|
||
alice, bob, eve, chat = c.bootstrap()
|
||
data = big_png()
|
||
check(len(data) <= 10 * 2**20, 'fixture exceeds 10 MiB')
|
||
results = []
|
||
for n in [int(x) for x in args.runs.split(',')]:
|
||
samplers = [Sampler(s) for s in SERVICES]
|
||
time.sleep(1.5)
|
||
|
||
def one(_):
|
||
local = Client(args.base_url, args.ca_file, timeout=120)
|
||
started = time.monotonic()
|
||
try:
|
||
r = local.call('POST', '/api/v1/chats/%s/attachments?filename=big.png' % chat['id'], [201, 503], actor=alice,
|
||
raw=data, headers={'Content-Type': 'image/png'})
|
||
return r['status'], time.monotonic() - started, r['headers'].get('X-Instance-Id')
|
||
except Failure as e:
|
||
return str(e), time.monotonic() - started, None
|
||
started = time.monotonic()
|
||
with ThreadPoolExecutor(max_workers=n) as pool:
|
||
done = list(pool.map(one, range(n)))
|
||
wall = time.monotonic() - started
|
||
time.sleep(1)
|
||
memory = {s.service: s.stop() for s in samplers}
|
||
statuses = {}
|
||
for status, _, _ in done:
|
||
statuses[str(status)] = statuses.get(str(status), 0) + 1
|
||
latencies = sorted(d for _, d, _ in done)
|
||
result = {'concurrent_uploads': n, 'file_mib': round(len(data) / 2**20, 2), 'statuses': statuses,
|
||
'replicas_used': len({i for _, _, i in done if i}), 'wall_seconds': round(wall, 2),
|
||
'latency_seconds': {'p50': round(latencies[len(latencies) // 2], 2), 'max': round(latencies[-1], 2)},
|
||
'memory': memory}
|
||
print(json.dumps(result, ensure_ascii=False))
|
||
results.append(result)
|
||
report = {'experiment': 'concurrent ~10 MiB PNG uploads through nginx', 'api_memory_limit': '512M per replica, JVM heap max 128 MiB',
|
||
'max_concurrent_uploads_per_replica': 8, 'results': results}
|
||
if args.output:
|
||
Path(args.output).parent.mkdir(parents=True, exist_ok=True)
|
||
Path(args.output).write_text(json.dumps(report, ensure_ascii=False, indent=2) + '\n')
|
||
|
||
|
||
if __name__ == '__main__':
|
||
run(main)
|