231 lines
9.2 KiB
Python
231 lines
9.2 KiB
Python
"""Эксперимент: пик памяти API при N одновременных загрузках (stdlib + docker CLI).
|
|
|
|
Гипотеза: память реплики ограничена конструкцией, а не числом/размером
|
|
загрузок: тело пишется в SpooledTemporaryFile (в RAM <= UPLOAD_SPOOL_MEMORY_BYTES
|
|
на загрузку), PNG проверяется потоково, одновременно <= MAX_CONCURRENT_UPLOADS
|
|
на реплику. Ожидаем прирост пика RSS << N x размер файла.
|
|
|
|
Методика: перезапуск API-реплик (сброс high-water mark) -> базовые VmRSS/VmHWM ->
|
|
N потоков одновременно (Barrier) загружают валидный PNG из шума ~9.5 MiB ->
|
|
во время загрузок внутри каждой реплики каждые 100 мс читается VmRSS процесса
|
|
API (/proc/1/status) и anon cgroup (без page cache временных файлов) -> пик
|
|
минус базовое значение -> проверка SHA-256. VmHWM не годится: стартовый пик
|
|
импорта библиотек (~170 MiB) выше пика загрузок.
|
|
|
|
docker compose --profile tools run --rm tester python3 student-tests/upload_memory.py --concurrency 12
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import base64
|
|
import collections
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import struct
|
|
import subprocess
|
|
import threading
|
|
import time
|
|
import urllib.error
|
|
import urllib.request
|
|
import uuid
|
|
import zlib
|
|
|
|
PROJECT = os.environ.get('COMPOSE_PROJECT', 'messenger-lab4')
|
|
|
|
|
|
def docker(*args) -> str:
|
|
return subprocess.run(['docker', *args], check=True, capture_output=True, text=True).stdout.strip()
|
|
|
|
|
|
def service_containers(service: str) -> list[str]:
|
|
return docker('ps', '-q', '--filter', f'label=com.docker.compose.project={PROJECT}',
|
|
'--filter', f'label=com.docker.compose.service={service}').split()
|
|
|
|
|
|
def api_containers() -> list[str]:
|
|
return service_containers('api')
|
|
|
|
|
|
def memory(cid: str) -> dict:
|
|
out = docker('exec', cid, 'sh', '-c',
|
|
'grep -E "^Vm(RSS|HWM)" /proc/1/status; '
|
|
'echo "peak $(cat /sys/fs/cgroup/memory.peak 2>/dev/null || echo 0)"; '
|
|
'echo "current $(cat /sys/fs/cgroup/memory.current)"; '
|
|
'grep -E "^anon " /sys/fs/cgroup/memory.stat')
|
|
values = {}
|
|
for line in out.splitlines():
|
|
name, value = line.replace(':', '').split()[:2]
|
|
values[name] = int(value) * (1024 if name.startswith('Vm') else 1)
|
|
return {k: round(v / 2 ** 20, 1) for k, v in values.items()}
|
|
|
|
|
|
SAMPLER = ('while :; do r=$(grep VmRSS /proc/1/status | tr -s " " | cut -d" " -f2); '
|
|
'a=$(grep "^anon " /sys/fs/cgroup/memory.stat | cut -d" " -f2); '
|
|
'echo "$r $a"; sleep 0.1; done')
|
|
|
|
|
|
class Sampler:
|
|
def __init__(self, cid: str):
|
|
self.proc = subprocess.Popen(['docker', 'exec', cid, 'sh', '-c', SAMPLER],
|
|
stdout=subprocess.PIPE, text=True)
|
|
self.rss = self.anon = 0
|
|
self.samples = 0
|
|
self.thread = threading.Thread(target=self._read, daemon=True)
|
|
self.thread.start()
|
|
|
|
def _read(self):
|
|
for line in self.proc.stdout:
|
|
try:
|
|
rss_kib, anon = map(int, line.split())
|
|
except ValueError:
|
|
continue
|
|
self.rss = max(self.rss, rss_kib * 1024)
|
|
self.anon = max(self.anon, anon)
|
|
self.samples += 1
|
|
|
|
def stop(self) -> dict:
|
|
self.proc.terminate()
|
|
self.proc.wait()
|
|
return {'VmRSS_peak': round(self.rss / 2 ** 20, 1), 'anon_peak': round(self.anon / 2 ** 20, 1),
|
|
'samples': self.samples}
|
|
|
|
|
|
def noise_png(target_bytes: int) -> bytes:
|
|
width = 1800
|
|
height = target_bytes // (width * 3 + 1)
|
|
raw = b''.join(b'\x00' + os.urandom(width * 3) for _ in range(height))
|
|
def chunk(kind, data):
|
|
return struct.pack('>I', len(data)) + kind + data + struct.pack('>I', zlib.crc32(kind + data))
|
|
ihdr = struct.pack('>IIBBBBB', width, height, 8, 2, 0, 0, 0)
|
|
return b'\x89PNG\r\n\x1a\n' + chunk(b'IHDR', ihdr) + chunk(b'IDAT', zlib.compress(raw, 1)) + chunk(b'IEND', b'')
|
|
|
|
|
|
def call(base, method, path, body=None, auth=None, raw=None, ctype=None, timeout=120):
|
|
h = {}
|
|
if auth:
|
|
h['Authorization'] = auth
|
|
if body is not None:
|
|
raw, ctype = json.dumps(body).encode(), 'application/json'
|
|
if ctype:
|
|
h['Content-Type'] = ctype
|
|
req = urllib.request.Request(base + '/api/v1' + path, data=raw, headers=h, method=method)
|
|
try:
|
|
with urllib.request.urlopen(req, timeout=timeout) as r:
|
|
return r.status, r.read(), r.headers.get('X-Instance-Id')
|
|
except urllib.error.HTTPError as e:
|
|
return e.code, e.read(), e.headers.get('X-Instance-Id')
|
|
|
|
|
|
def wait_ready(base, seconds=90):
|
|
deadline = time.monotonic() + seconds
|
|
while time.monotonic() < deadline:
|
|
try:
|
|
with urllib.request.urlopen(base + '/health/ready', timeout=2) as r:
|
|
if r.status == 200:
|
|
return
|
|
except Exception:
|
|
pass
|
|
time.sleep(1)
|
|
raise SystemExit('API is not ready')
|
|
|
|
|
|
def main() -> int:
|
|
p = argparse.ArgumentParser()
|
|
p.add_argument('--base-url', default=os.environ.get('BASE_URL', 'http://lb:8080'))
|
|
p.add_argument('--concurrency', type=int, default=12)
|
|
p.add_argument('--size-mib', type=float, default=9.5)
|
|
p.add_argument('--no-restart', action='store_true')
|
|
p.add_argument('--output')
|
|
args = p.parse_args()
|
|
base = args.base_url.rstrip('/')
|
|
|
|
data = noise_png(int(args.size_mib * 2 ** 20))
|
|
digest = hashlib.sha256(data).hexdigest()
|
|
print(f'file: {len(data)} bytes ({len(data) / 2 ** 20:.2f} MiB), concurrency {args.concurrency}', flush=True)
|
|
|
|
containers = api_containers()
|
|
if not args.no_restart:
|
|
docker('restart', '--time', '15', *containers)
|
|
time.sleep(3)
|
|
wait_ready(base)
|
|
time.sleep(3)
|
|
|
|
name = 'mem' + uuid.uuid4().hex[:12]
|
|
password = 'upload-memory-password'
|
|
call(base, 'POST', '/users', {'username': name, 'display_name': 'Mem', 'password': password})
|
|
basic = 'Basic ' + base64.b64encode(f'{name}:{password}'.encode()).decode()
|
|
s, session, _ = call(base, 'POST', '/auth/sessions', auth=basic)
|
|
auth = 'Bearer ' + json.loads(session)['token']
|
|
user = json.loads(call(base, 'GET', '/users/me', auth=auth)[1])
|
|
chat = json.loads(call(base, 'POST', '/chats', {'title': 'mem', 'member_ids': [user['id']]}, auth=auth)[1])
|
|
|
|
before = {c[:12]: memory(c) for c in containers}
|
|
samplers = {c[:12]: Sampler(c) for c in containers}
|
|
s3 = service_containers('s3')[0]
|
|
s3_before = memory(s3)
|
|
s3_sampler = Sampler(s3)
|
|
time.sleep(1)
|
|
barrier = threading.Barrier(args.concurrency)
|
|
results = []
|
|
lock = threading.Lock()
|
|
|
|
def worker():
|
|
barrier.wait()
|
|
t = time.monotonic()
|
|
s, body, instance = call(base, 'POST', f"/chats/{chat['id']}/attachments?filename=noise.png",
|
|
auth=auth, raw=data, ctype='image/png')
|
|
with lock:
|
|
results.append((s, time.monotonic() - t, instance, body))
|
|
|
|
started = time.monotonic()
|
|
threads = [threading.Thread(target=worker) for _ in range(args.concurrency)]
|
|
for t in threads:
|
|
t.start()
|
|
for t in threads:
|
|
t.join()
|
|
wall = time.monotonic() - started
|
|
time.sleep(0.5)
|
|
peaks = {c: sm.stop() for c, sm in samplers.items()}
|
|
s3_peak = s3_sampler.stop()
|
|
after = {c[:12]: memory(c) for c in containers}
|
|
|
|
ok = [r for r in results if r[0] == 201]
|
|
sha_ok = all(json.loads(r[3])['sha256'] == digest for r in ok)
|
|
if ok:
|
|
att = json.loads(ok[0][3])
|
|
s, content, _ = call(base, 'GET', f"/attachments/{att['id']}/content", auth=auth)
|
|
sha_ok = sha_ok and hashlib.sha256(content).hexdigest() == digest
|
|
latencies = sorted(r[1] for r in ok)
|
|
summary = {
|
|
'concurrency': args.concurrency,
|
|
'file_bytes': len(data),
|
|
'statuses': dict(collections.Counter(r[0] for r in results)),
|
|
'per_instance': dict(collections.Counter(r[2] for r in results)),
|
|
'wall_seconds': round(wall, 2),
|
|
'latency_p50_s': round(latencies[len(latencies) // 2], 2) if latencies else None,
|
|
'latency_max_s': round(latencies[-1], 2) if latencies else None,
|
|
'sha256_ok': sha_ok,
|
|
'memory_mib_before': before,
|
|
'memory_mib_peak_during': peaks,
|
|
'memory_mib_after': after,
|
|
'rss_peak_increase_mib': {
|
|
c: round(peaks[c]['VmRSS_peak'] - before[c]['VmRSS'], 1) for c in peaks
|
|
},
|
|
'anon_peak_increase_mib': {
|
|
c: round(peaks[c]['anon_peak'] - before[c]['anon'], 1) for c in peaks
|
|
},
|
|
's3_server_mib': {'VmRSS_before': s3_before['VmRSS'], **s3_peak},
|
|
'naive_in_memory_estimate_mib': round(args.concurrency * len(data) / 2 ** 20, 1),
|
|
}
|
|
text = json.dumps(summary, ensure_ascii=False, indent=2)
|
|
print(text)
|
|
if args.output:
|
|
with open(args.output, 'w', encoding='utf-8') as f:
|
|
f.write(text + '\n')
|
|
return 0 if ok and sha_ok and all(r[0] in (201, 429) for r in results) else 1
|
|
|
|
|
|
if __name__ == '__main__':
|
|
raise SystemExit(main())
|