#!/usr/bin/env python3 """Эксперимент уровня 4: N одновременных загрузок ~10 MiB и пик памяти контейнеров. Память читается из cgroup v2 внутри контейнеров (memory.current и anon из memory.stat) каждые 0.2 с через docker compose exec, пока идут загрузки. anon — память процессов (heap JVM и native), memory.current дополнительно включает page cache временных файлов. Пример: python3 tests/upload_memory.py --runs 1,10,20 --output evidence/upload_memory.json """ import json import os import struct import subprocess import threading import time import zlib from concurrent.futures import ThreadPoolExecutor from pathlib import Path from client import ROOT, Client, Failure, arguments, check, run SERVICES = ('api-1', 'api-2', 'nginx', 's3') # Свой дедлайн: docker exec не завершает процесс в контейнере, когда клиент отключается. SAMPLER = ('end=$(($(date +%s)+180)); while [ $(date +%s) -lt $end ]; do ' 'echo $(cat /sys/fs/cgroup/memory.current) $(grep "^anon " /sys/fs/cgroup/memory.stat | cut -d" " -f2) || exit; sleep 0.2; done') def big_png(): # 1800×1900 RGB из случайных байтов не сжимается: файл ≈9.8 MiB, чуть меньше лимита 10 MiB. def chunk(kind, data): return struct.pack('>I', len(data)) + kind + data + struct.pack('>I', zlib.crc32(kind + data)) raw = b''.join(b'\x00' + os.urandom(1800 * 3) for _ in range(1900)) return (b'\x89PNG\r\n\x1a\n' + chunk(b'IHDR', struct.pack('>IIBBBBB', 1800, 1900, 8, 2, 0, 0, 0)) + chunk(b'IDAT', zlib.compress(raw, 1)) + chunk(b'IEND', b'')) class Sampler: """Фоновое чтение памяти одного контейнера; хранит максимум за время измерения.""" def __init__(self, service): self.service, self.peak_current, self.peak_anon, self.base = service, 0, 0, None self.proc = subprocess.Popen(['docker', 'compose', 'exec', '-T', service, 'sh', '-c', SAMPLER], cwd=ROOT, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True) threading.Thread(target=self.read, daemon=True).start() def read(self): for line in self.proc.stdout: parts = line.split() if len(parts) != 2: continue current, anon = map(int, parts) if self.base is None: self.base = (current, anon) self.peak_current, self.peak_anon = max(self.peak_current, current), max(self.peak_anon, anon) def stop(self): self.proc.terminate() mib = lambda v: round(v / 2**20, 1) return {'baseline_current_mib': mib(self.base[0]), 'baseline_anon_mib': mib(self.base[1]), 'peak_current_mib': mib(self.peak_current), 'peak_anon_mib': mib(self.peak_anon)} def main(): def configure(p): p.add_argument('--runs', default='1,10,20', help='comma-separated concurrency levels') p.add_argument('--output', default=None) args = arguments('Concurrent ~10 MiB uploads with peak container memory', configure) c = Client(args.base_url, args.ca_file, timeout=120) alice, bob, eve, chat = c.bootstrap() data = big_png() check(len(data) <= 10 * 2**20, 'fixture exceeds 10 MiB') results = [] for n in [int(x) for x in args.runs.split(',')]: samplers = [Sampler(s) for s in SERVICES] time.sleep(1.5) def one(_): local = Client(args.base_url, args.ca_file, timeout=120) started = time.monotonic() try: r = local.call('POST', '/api/v1/chats/%s/attachments?filename=big.png' % chat['id'], [201, 503], actor=alice, raw=data, headers={'Content-Type': 'image/png'}) return r['status'], time.monotonic() - started, r['headers'].get('X-Instance-Id') except Failure as e: return str(e), time.monotonic() - started, None started = time.monotonic() with ThreadPoolExecutor(max_workers=n) as pool: done = list(pool.map(one, range(n))) wall = time.monotonic() - started time.sleep(1) memory = {s.service: s.stop() for s in samplers} statuses = {} for status, _, _ in done: statuses[str(status)] = statuses.get(str(status), 0) + 1 latencies = sorted(d for _, d, _ in done) result = {'concurrent_uploads': n, 'file_mib': round(len(data) / 2**20, 2), 'statuses': statuses, 'replicas_used': len({i for _, _, i in done if i}), 'wall_seconds': round(wall, 2), 'latency_seconds': {'p50': round(latencies[len(latencies) // 2], 2), 'max': round(latencies[-1], 2)}, 'memory': memory} print(json.dumps(result, ensure_ascii=False)) results.append(result) report = {'experiment': 'concurrent ~10 MiB PNG uploads through nginx', 'api_memory_limit': '512M per replica, JVM heap max 128 MiB', 'max_concurrent_uploads_per_replica': 8, 'results': results} if args.output: Path(args.output).parent.mkdir(parents=True, exist_ok=True) Path(args.output).write_text(json.dumps(report, ensure_ascii=False, indent=2) + '\n') if __name__ == '__main__': run(main)