#!/usr/bin/env python3 import argparse import datetime import hashlib import http.client import json import math import os from pathlib import Path import re import struct import subprocess import sys import threading import time import zlib ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT / 'tests')) from client import Client HOST = os.environ.get('LB_HOST', 'lb') PORT = int(os.environ.get('LB_PORT', '8080')) SAMPLE = (ROOT / 'tests/fixtures/sample.png').read_bytes() MiB = 1024 * 1024 def docker(*args, text=True): return subprocess.run(['docker', *args], check=True, capture_output=True, text=text).stdout def replicas(): return [cid[:12] for cid in docker('compose', 'ps', '-q', 'api').split()] def chunk(kind, data): return struct.pack('>I', len(data)) + kind + data + struct.pack('>I', zlib.crc32(kind + data) & 0xffffffff) def padded_png(size): return SAMPLE[:33] + chunk(b'abCd', b'\x01' * (size - len(SAMPLE) - 12)) + SAMPLE[33:] def big_png(width, height): rows = (b'\0' + b'\x55' * (width * 3)) * height header = struct.pack('>IIBBBBB', width, height, 8, 2, 0, 0, 0) return b'\x89PNG\r\n\x1a\n' + chunk(b'IHDR', header) + chunk(b'IDAT', zlib.compress(rows, 9)) + chunk(b'IEND', b'') def big_jpeg(width, height): script = ( "require('sharp')({create:{width:%d,height:%d,channels:3,background:'#336699'}})" ".jpeg({quality:80}).toBuffer().then(b=>process.stdout.write(b))" % (width, height) ) return docker('exec', replicas()[0], 'node', '-e', script, text=False) def cgroup_peak(cid): raw = docker('exec', cid, 'sh', '-c', 'cat /sys/fs/cgroup/memory/memory.max_usage_in_bytes 2>/dev/null || cat /sys/fs/cgroup/memory.peak 2>/dev/null || echo 0') return int(raw.strip()) // MiB def percentile(values, q): ordered = sorted(values) return round(ordered[max(0, math.ceil(q * len(ordered)) - 1)], 3) class Sampler: def __init__(self, cids): self.rss = {cid: 0 for cid in cids} self.cpu = {} self.stop_flag = threading.Event() self.threads = [threading.Thread(target=self.watch, args=(cid,)) for cid in cids] self.threads.append(threading.Thread(target=self.watch_cpu)) def watch(self, cid): script = 'while :; do grep VmRSS /proc/1/status; sleep 0.2; done' process = subprocess.Popen(['docker', 'exec', cid, 'sh', '-c', script], stdout=subprocess.PIPE, text=True) for line in process.stdout: if self.stop_flag.is_set(): break match = re.match(r'VmRSS:\s+(\d+)', line) if match: self.rss[cid] = max(self.rss[cid], int(match.group(1)) // 1024) process.kill() process.wait() def watch_cpu(self): while not self.stop_flag.is_set(): lines = docker('stats', '--no-stream', '--format', '{{.Name}} {{.CPUPerc}} {{.MemUsage}}') for line in lines.splitlines(): name, cpu, usage = line.split(' ', 2) value = float(cpu.rstrip('%')) entry = self.cpu.setdefault(name, {'max_cpu_percent': 0, 'last_mem': ''}) entry['max_cpu_percent'] = max(entry['max_cpu_percent'], value) entry['last_mem'] = usage.split(' / ')[0] def __enter__(self): [thread.start() for thread in self.threads] return self def __exit__(self, *_): self.stop_flag.set() for thread in self.threads: thread.join(timeout=5) def restart_replicas(): docker('compose', 'restart', 'api') deadline = time.monotonic() + 90 expected = set(replicas()) while time.monotonic() < deadline: seen = set() for _ in range(8): try: conn = http.client.HTTPConnection(HOST, PORT, timeout=3) conn.request('GET', '/health/ready') response = conn.getresponse() response.read() if response.status == 200: seen.add(response.getheader('X-Instance-Id')) conn.close() except OSError: pass if seen >= expected: return time.sleep(0.5) raise SystemExit('replicas did not return') def upload(path, token, body, content_type): conn = http.client.HTTPConnection(HOST, PORT, timeout=120) started = time.monotonic() conn.request('POST', path, body=body, headers={'Authorization': 'Bearer ' + token, 'Content-Type': content_type}) response = conn.getresponse() data = response.read() elapsed = time.monotonic() - started conn.close() return response.status, elapsed, json.loads(data), response.getheader('X-Instance-Id') def download_sha(attachment_id, token): conn = http.client.HTTPConnection(HOST, PORT, timeout=60) conn.request('GET', '/api/v1/attachments/' + attachment_id + '/content', headers={'Authorization': 'Bearer ' + token}) response = conn.getresponse() data = response.read() conn.close() return response.status, hashlib.sha256(data).hexdigest() def run_profile(name, body, content_type, concurrency, rounds, token, chat): restart_replicas() cids = replicas() idle = {cid: int(re.search(r'VmRSS:\s+(\d+)', docker('exec', cid, 'cat', '/proc/1/status')).group(1)) // 1024 for cid in cids} path = '/api/v1/chats/' + chat + '/attachments?filename=load.bin' expected_sha = hashlib.sha256(body).hexdigest() results = [] lock = threading.Lock() def worker(): try: outcome = upload(path, token, body, content_type) except OSError as error: outcome = (0, 0.0, {'error': {'code': type(error).__name__}}, None) with lock: results.append(outcome) with Sampler(cids) as sampler: started = time.monotonic() for _ in range(rounds): threads = [threading.Thread(target=worker) for _ in range(concurrency)] [thread.start() for thread in threads] [thread.join() for thread in threads] wall = time.monotonic() - started time.sleep(0.5) created = [item for item in results if item[0] == 201] latencies = [item[1] for item in created] statuses = {} for item in results: statuses[str(item[0])] = statuses.get(str(item[0]), 0) + 1 sha_ok = all(item[2]['sha256'] == expected_sha for item in created) stored_ok = None if created: status, sha = download_sha(created[0][2]['id'], token) stored_ok = status == 200 and sha == expected_sha per_replica = {} for cid in cids: info = docker('exec', cid, 'cat', '/proc/1/status') hwm = int(re.search(r'VmHWM:\s+(\d+)', info).group(1)) // 1024 oom = docker('inspect', '-f', '{{.State.OOMKilled}}', cid).strip() temp = docker('exec', cid, 'sh', '-c', 'ls /tmp | grep -c messenger-upload || true').strip() per_replica[cid] = { 'idle_rss_mib': idle[cid], 'sampled_peak_rss_mib': sampler.rss[cid], 'vm_hwm_mib': hwm, 'cgroup_max_usage_with_page_cache_mib': cgroup_peak(cid), 'oom_killed': oom, 'temp_dirs_left': int(temp or 0), 'served': sum(1 for item in results if item[3] == cid), } return { 'profile': name, 'body_bytes': len(body), 'content_type': content_type, 'concurrency': concurrency, 'rounds': rounds, 'requests': len(results), 'statuses': statuses, 'wall_seconds': round(wall, 2), 'successful_rps': round(len(created) / wall, 2), 'error_rate': round(1 - len(created) / len(results), 4), 'latency_seconds': None if not latencies else { 'p50': percentile(latencies, 0.5), 'p95': percentile(latencies, 0.95), 'p99': percentile(latencies, 0.99), }, 'sha256_in_responses_ok': sha_ok, 'download_sha256_ok': stored_ok, 'replicas': per_replica, 'docker_stats': sampler.cpu, } def main(): parser = argparse.ArgumentParser() parser.add_argument('--output') parser.add_argument('--rounds', type=int, default=3) args = parser.parse_args() client = Client('http://%s:%d' % (HOST, PORT)) alice = client.user('upload_load') chat = client.call('POST', '/api/v1/chats', 201, actor=alice, body={'title': 'upload experiment', 'member_ids': [alice['id']]}, schema='Chat')['json'] token, chat_id = alice['token'], chat['id'] jpeg_40mp = big_jpeg(8000, 5000) png_40mp = big_png(8000, 5000) profiles = [ ('sample 112 KiB x10', SAMPLE, 'image/png', 10), ('png 10 MiB x1', padded_png(10 * MiB), 'image/png', 1), ('png 10 MiB x10', padded_png(10 * MiB), 'image/png', 10), ('png 40 MP x10', png_40mp, 'image/png', 10), ('jpeg 40 MP x10', jpeg_40mp, 'image/jpeg', 10), ('png 10 MiB x20', padded_png(10 * MiB), 'image/png', 20), ('png 10 MiB x40 (выше лимита 16 на реплику)', padded_png(10 * MiB), 'image/png', 40), ] results = [] for name, body, content_type, concurrency in profiles: print('profile:', name, flush=True) results.append(run_profile(name, body, content_type, concurrency, args.rounds, token, chat_id)) count = docker('compose', 'exec', '-T', 'postgres', 'psql', '-U', 'messenger', '-d', 'messenger', '-tAc', 'SELECT (SELECT count(*) FROM attachments), (SELECT count(*) FROM messages)').strip() limits = docker('inspect', '-f', '{{.HostConfig.NanoCpus}} {{.HostConfig.Memory}}', replicas()[0]).split() report = { 'date': datetime.datetime.now(datetime.timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ'), 'api_replica_limits': {'cpus': int(limits[0]) / 1e9, 'memory_mib': int(limits[1]) // MiB}, 'rows_after_attachments_messages': count, 'results': results, } text = json.dumps(report, ensure_ascii=False, indent=2) if args.output: Path(args.output).write_text(text + '\n', encoding='utf-8') print(text) if __name__ == '__main__': main()