Files
mtusi2026lab4/tests/upload_memory.py
T

108 lines
5.4 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env python3
"""Эксперимент уровня 4: N одновременных загрузок ~10 MiB и пик памяти контейнеров.
Память читается из cgroup v2 внутри контейнеров (memory.current и anon из memory.stat) каждые 0.2 с
через docker compose exec, пока идут загрузки. anon — память процессов (heap JVM и native),
memory.current дополнительно включает page cache временных файлов.
Пример: python3 tests/upload_memory.py --runs 1,10,20 --output evidence/upload_memory.json
"""
import json
import os
import struct
import subprocess
import threading
import time
import zlib
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
from client import ROOT, Client, Failure, arguments, check, run
SERVICES = ('api-1', 'api-2', 'nginx', 's3')
# Свой дедлайн: docker exec не завершает процесс в контейнере, когда клиент отключается.
SAMPLER = ('end=$(($(date +%s)+180)); while [ $(date +%s) -lt $end ]; do '
'echo $(cat /sys/fs/cgroup/memory.current) $(grep "^anon " /sys/fs/cgroup/memory.stat | cut -d" " -f2) || exit; sleep 0.2; done')
def big_png():
# 1800×1900 RGB из случайных байтов не сжимается: файл ≈9.8 MiB, чуть меньше лимита 10 MiB.
def chunk(kind, data):
return struct.pack('>I', len(data)) + kind + data + struct.pack('>I', zlib.crc32(kind + data))
raw = b''.join(b'\x00' + os.urandom(1800 * 3) for _ in range(1900))
return (b'\x89PNG\r\n\x1a\n' + chunk(b'IHDR', struct.pack('>IIBBBBB', 1800, 1900, 8, 2, 0, 0, 0))
+ chunk(b'IDAT', zlib.compress(raw, 1)) + chunk(b'IEND', b''))
class Sampler:
"""Фоновое чтение памяти одного контейнера; хранит максимум за время измерения."""
def __init__(self, service):
self.service, self.peak_current, self.peak_anon, self.base = service, 0, 0, None
self.proc = subprocess.Popen(['docker', 'compose', 'exec', '-T', service, 'sh', '-c', SAMPLER], cwd=ROOT,
stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True)
threading.Thread(target=self.read, daemon=True).start()
def read(self):
for line in self.proc.stdout:
parts = line.split()
if len(parts) != 2:
continue
current, anon = map(int, parts)
if self.base is None:
self.base = (current, anon)
self.peak_current, self.peak_anon = max(self.peak_current, current), max(self.peak_anon, anon)
def stop(self):
self.proc.terminate()
mib = lambda v: round(v / 2**20, 1)
return {'baseline_current_mib': mib(self.base[0]), 'baseline_anon_mib': mib(self.base[1]),
'peak_current_mib': mib(self.peak_current), 'peak_anon_mib': mib(self.peak_anon)}
def main():
def configure(p):
p.add_argument('--runs', default='1,10,20', help='comma-separated concurrency levels')
p.add_argument('--output', default=None)
args = arguments('Concurrent ~10 MiB uploads with peak container memory', configure)
c = Client(args.base_url, args.ca_file, timeout=120)
alice, bob, eve, chat = c.bootstrap()
data = big_png()
check(len(data) <= 10 * 2**20, 'fixture exceeds 10 MiB')
results = []
for n in [int(x) for x in args.runs.split(',')]:
samplers = [Sampler(s) for s in SERVICES]
time.sleep(1.5)
def one(_):
local = Client(args.base_url, args.ca_file, timeout=120)
started = time.monotonic()
try:
r = local.call('POST', '/api/v1/chats/%s/attachments?filename=big.png' % chat['id'], [201, 503], actor=alice,
raw=data, headers={'Content-Type': 'image/png'})
return r['status'], time.monotonic() - started, r['headers'].get('X-Instance-Id')
except Failure as e:
return str(e), time.monotonic() - started, None
started = time.monotonic()
with ThreadPoolExecutor(max_workers=n) as pool:
done = list(pool.map(one, range(n)))
wall = time.monotonic() - started
time.sleep(1)
memory = {s.service: s.stop() for s in samplers}
statuses = {}
for status, _, _ in done:
statuses[str(status)] = statuses.get(str(status), 0) + 1
latencies = sorted(d for _, d, _ in done)
result = {'concurrent_uploads': n, 'file_mib': round(len(data) / 2**20, 2), 'statuses': statuses,
'replicas_used': len({i for _, _, i in done if i}), 'wall_seconds': round(wall, 2),
'latency_seconds': {'p50': round(latencies[len(latencies) // 2], 2), 'max': round(latencies[-1], 2)},
'memory': memory}
print(json.dumps(result, ensure_ascii=False))
results.append(result)
report = {'experiment': 'concurrent ~10 MiB PNG uploads through nginx', 'api_memory_limit': '512M per replica, JVM heap max 128 MiB',
'max_concurrent_uploads_per_replica': 8, 'results': results}
if args.output:
Path(args.output).parent.mkdir(parents=True, exist_ok=True)
Path(args.output).write_text(json.dumps(report, ensure_ascii=False, indent=2) + '\n')
if __name__ == '__main__':
run(main)