#!/usr/bin/env python3 """Compare API replica counts under fixed and increasing total API resources.""" import argparse import json import os from pathlib import Path import platform import re import shlex import subprocess import sys import tempfile import time ROOT = Path(__file__).resolve().parents[1] BASE_URL = os.environ.get('BASE_URL', 'http://localhost:8080') COMPOSE = shlex.split(os.environ.get('COMPOSE', 'docker compose')) LOAD_SCRIPT = ROOT / 'tests' / 'load.py' MEMORY_PATTERN = re.compile(r'([\d.]+)\s*(B|KiB|MiB|GiB|kB|MB|GB)', re.IGNORECASE) os.environ.setdefault('PYTHONUTF8', '1') def command(args, overlay=None, check=True, capture=False): argv = COMPOSE + ['--env-file', str(ROOT / '.env')] if overlay: argv += ['--env-file', str(overlay)] argv += ['-f', str(ROOT / 'compose.yaml')] + list(args) result = subprocess.run( argv, cwd=ROOT, check=False, text=True, capture_output=capture, ) if check and result.returncode: detail = (result.stderr or result.stdout or '').strip() raise RuntimeError('Compose command failed: ' + ' '.join(args) + '\n' + detail) return result def wait_ready(timeout=90): from urllib.error import URLError from urllib.request import urlopen deadline = time.monotonic() + timeout while time.monotonic() < deadline: try: with urlopen(BASE_URL.rstrip('/') + '/health/ready', timeout=2) as response: if response.status == 200: return except (OSError, URLError): time.sleep(0.5) raise TimeoutError('Load balancer did not become ready within %s seconds' % timeout) def sample_instances(minimum, timeout=15): sys.path.insert(0, str(ROOT / 'tests')) from client import Client, Failure client = Client(BASE_URL, timeout=5) instances = set() deadline = time.monotonic() + timeout while time.monotonic() < deadline: for _ in range(40): try: response = client.call( 'GET', '/api/v1/users/me', expected=401, schema='Error', ) instances.add(response['headers']['X-Instance-Id']) except Failure: time.sleep(0.1) if len(instances) >= minimum: return instances time.sleep(1) raise TimeoutError( 'Expected %d API instance IDs through Nginx; observed %d' % (minimum, len(instances)) ) def parse_memory(value): match = MEMORY_PATTERN.search(value) if not match: return 0 amount = float(match.group(1)) unit = match.group(2).lower() factors = { 'b': 1, 'kib': 1024, 'mib': 1024 ** 2, 'gib': 1024 ** 3, 'kb': 1000, 'mb': 1000 ** 2, 'gb': 1000 ** 3, } return int(amount * factors[unit]) def collect_stats(): result = command(['ps', '-q'], check=True, capture=True) container_ids = result.stdout.split() if not container_ids: return {} stats = subprocess.run( [ 'docker', 'stats', '--no-stream', '--format', '{{.Name}};{{.CPUPerc}};{{.MemUsage}}', *container_ids, ], cwd=ROOT, text=True, capture_output=True, check=False, ) if stats.returncode: raise RuntimeError('docker stats failed: ' + (stats.stderr or stats.stdout).strip()) values = {} for line in stats.stdout.splitlines(): parts = line.split(';', 2) if len(parts) != 3: continue name, cpu, memory = parts service = next( (item for item in ('api', 'postgres', 'nginx') if '-' + item + '-' in name), None, ) if service: values.setdefault(service, []).append( { 'cpu_percent': float(cpu.rstrip('%') or 0), 'memory_bytes': parse_memory(memory.split('/')[0]), } ) return values def run_load(duration, concurrency, output=None): args = [ sys.executable, str(LOAD_SCRIPT), '--base-url', BASE_URL, '--duration', str(duration), '--concurrency', str(concurrency), ] if output: args += ['--output', str(output)] return subprocess.run( args, cwd=ROOT, text=True, capture_output=True, check=False, ) def measure(duration, concurrency, output): process = subprocess.Popen( [ sys.executable, str(LOAD_SCRIPT), '--base-url', BASE_URL, '--duration', str(duration), '--concurrency', str(concurrency), '--output', str(output), ], cwd=ROOT, text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, ) peaks = {} try: while process.poll() is None: for service, entries in collect_stats().items(): values = peaks.setdefault( service, {'cpu_percent': 0.0, 'memory_bytes': 0} ) values['cpu_percent'] = max( values['cpu_percent'], sum(entry['cpu_percent'] for entry in entries), ) values['memory_bytes'] = max( values['memory_bytes'], sum(entry['memory_bytes'] for entry in entries), ) time.sleep(1) stdout, stderr = process.communicate() finally: if process.poll() is None: process.terminate() process.wait(timeout=5) if process.returncode: raise RuntimeError('Load run failed:\n' + (stderr or stdout).strip()) return json.loads(output.read_text(encoding='utf-8')), peaks def run_scenario(name, replicas, cpu_per_replica, memory_per_replica, args, output_dir): with tempfile.NamedTemporaryFile( mode='w', encoding='ascii', suffix='.env', prefix='lab3-track1-', delete=False, ) as handle: handle.write('API_REPLICAS=%s\n' % replicas) handle.write('API_CPU_LIMIT=%s\n' % cpu_per_replica) handle.write('API_MEMORY_LIMIT=%sM\n' % memory_per_replica) override = Path(handle.name) try: command( ['up', '--detach', '--build', '--scale', 'api=%d' % replicas], overlay=override, capture=True, ) wait_ready() instances = sample_instances(min(2, replicas)) running = command(['ps', '-q', 'api'], capture=True).stdout.split() if len(running) != replicas: raise RuntimeError( 'Expected %d running API replicas, found %d' % (replicas, len(running)) ) warmup = run_load(args.warmup, args.concurrency) if warmup.returncode: raise RuntimeError('Warm-up load failed: ' + warmup.stderr.strip()) runs = [] for repeat in range(1, args.repeats + 1): raw_path = output_dir / ('%s-run-%02d.json' % (name, repeat)) metrics, peaks = measure( args.duration, args.concurrency, raw_path, ) runs.append({'load': metrics, 'peak_resources': peaks, 'raw_file': raw_path.name}) return { 'name': name, 'api_replicas': replicas, 'api_cpu_limit_per_replica': cpu_per_replica, 'api_memory_limit_mib_per_replica': memory_per_replica, 'total_api_cpu_limit': round(replicas * cpu_per_replica, 3), 'total_api_memory_limit_mib': replicas * memory_per_replica, 'instances_observed': len(instances), 'runs': runs, } finally: override.unlink(missing_ok=True) def mean(values): return sum(values) / len(values) if values else 0 def summarize(scenario): runs = scenario['runs'] return { 'name': scenario['name'], 'api_replicas': scenario['api_replicas'], 'total_api_cpu_limit': scenario['total_api_cpu_limit'], 'total_api_memory_limit_mib': scenario['total_api_memory_limit_mib'], 'successful_rps_mean': round( mean([run['load']['successful_rps'] for run in runs]), 3 ), 'error_ratio_mean': round( mean([run['load']['error_ratio'] or 0 for run in runs]), 6 ), 'p95_latency_ms_mean': round( mean([run['load']['successful_latency_ms']['p95'] or 0 for run in runs]), 3, ), 'api_cpu_peak_percent_mean': round( mean([run['peak_resources'].get('api', {}).get('cpu_percent', 0) for run in runs]), 3, ), 'api_memory_peak_mib_mean': round( mean( [ run['peak_resources'].get('api', {}).get('memory_bytes', 0) / (1024 ** 2) for run in runs ] ), 3, ), 'postgres_cpu_peak_percent_mean': round( mean( [ run['peak_resources'].get('postgres', {}).get('cpu_percent', 0) for run in runs ] ), 3, ), 'postgres_memory_peak_mib_mean': round( mean( [ run['peak_resources'].get('postgres', {}).get('memory_bytes', 0) / (1024 ** 2) for run in runs ] ), 3, ), 'nginx_cpu_peak_percent_mean': round( mean([run['peak_resources'].get('nginx', {}).get('cpu_percent', 0) for run in runs]), 3, ), 'nginx_memory_peak_mib_mean': round( mean( [ run['peak_resources'].get('nginx', {}).get('memory_bytes', 0) / (1024 ** 2) for run in runs ] ), 3, ), } def write_chart(path, summaries, value_key, title, unit): width, height = 980, 430 left, top, chart_width, chart_height = 70, 55, 850, 275 maximum = max((item[value_key] for item in summaries), default=1) or 1 parts = [ '' % (width, height, width, height), '', '%s' % (left, title), ] for tick in range(5): y = top + chart_height - chart_height * tick / 4 value = maximum * tick / 4 parts.append( '' % (left, y, left + chart_width, y) ) parts.append( '%.1f' % (y + 4, value) ) bar_width = 72 slot = chart_width / len(summaries) for index, item in enumerate(summaries): value = item[value_key] bar_height = chart_height * value / maximum x = left + slot * index + (slot - bar_width) / 2 y = top + chart_height - bar_height color = '#2864a5' if item['name'].startswith('fixed') else '#d47a22' label = item['name'].replace('fixed-total-', 'fixed ').replace('growing-total-', 'growth ') parts.append( '' % (x, y, bar_width, bar_height, color) ) parts.append( '%.1f' % (x + bar_width / 2, max(top + 14, y - 6), value) ) parts.append( '%s' % (x + bar_width / 2, top + chart_height + 22, label) ) parts.extend( [ '%s' % (left, height - 14, unit), '', 'fixed API budget', '', 'growing API budget', '', ] ) path.write_text('\n'.join(parts) + '\n', encoding='utf-8') def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('--duration', type=int, default=30) parser.add_argument('--warmup', type=int, default=5) parser.add_argument('--concurrency', type=int, default=16) parser.add_argument('--repeats', type=int, default=3) parser.add_argument('--fixed-total-cpus', type=float, default=4.0) parser.add_argument('--fixed-total-memory-mib', type=int, default=2048) parser.add_argument('--growth-cpus-per-replica', type=float, default=1.0) parser.add_argument('--growth-memory-mib-per-replica', type=int, default=512) parser.add_argument('--output', default='evidence/track1/benchmark.json') args = parser.parse_args() if ( args.duration < 1 or args.warmup < 1 or args.concurrency < 1 or args.repeats < 1 or args.fixed_total_cpus <= 0 or args.fixed_total_memory_mib < 1 or args.growth_cpus_per_replica <= 0 or args.growth_memory_mib_per_replica < 1 ): parser.error('duration, warmup, concurrency, repeats and resource limits must be positive') if not (ROOT / '.env').is_file(): parser.error('Create .env and local Compose secrets first') if not all((ROOT / 'secrets' / name).is_file() for name in ('postgres_password', 'database_url')): parser.error('Run scripts/init-local-secrets.ps1 before benchmarking') output = (ROOT / args.output).resolve() output.parent.mkdir(parents=True, exist_ok=True) command(['config', '--quiet']) output_dir = output.parent scenarios = [] try: for replicas in (1, 2, 4): fixed_cpu = round(args.fixed_total_cpus / replicas, 3) fixed_memory = max(1, args.fixed_total_memory_mib // replicas) scenarios.append( run_scenario( 'fixed-total-%d' % replicas, replicas, fixed_cpu, fixed_memory, args, output_dir, ) ) for replicas in (1, 2, 4): scenarios.append( run_scenario( 'growing-total-%d' % replicas, replicas, args.growth_cpus_per_replica, args.growth_memory_mib_per_replica, args, output_dir, ) ) finally: command(['up', '--detach', '--build', '--scale', 'api=2'], capture=True) wait_ready() docker_version = subprocess.run( ['docker', 'version', '--format', '{{.Server.Version}}'], cwd=ROOT, text=True, capture_output=True, check=False, ) docker_host = subprocess.run( ['docker', 'info', '--format', '{{.NCPU}}|{{.MemTotal}}|{{.OperatingSystem}}'], cwd=ROOT, text=True, capture_output=True, check=False, ) docker_host_values = docker_host.stdout.strip().split('|') if docker_host.returncode or len(docker_host_values) != 3: raise RuntimeError('Unable to capture Docker host CPU/RAM information') report = { 'track': '1 — цена масштабирования', 'method': { 'workload': 'tests/load.py closed-loop POST messages through Nginx', 'duration_seconds_per_run': args.duration, 'warmup_seconds_per_scenario': args.warmup, 'concurrency': args.concurrency, 'repeats_per_scenario': args.repeats, 'fixed_total_api_limits': { 'cpus': args.fixed_total_cpus, 'memory_mib': args.fixed_total_memory_mib, }, 'growth_per_replica_limits': { 'cpus': args.growth_cpus_per_replica, 'memory_mib': args.growth_memory_mib_per_replica, }, 'host_cpu_count': os.cpu_count(), 'platform': platform.platform(), 'docker_server_version': docker_version.stdout.strip(), 'docker_host': { 'cpus': int(docker_host_values[0]), 'memory_mib': round(int(docker_host_values[1]) / (1024 ** 2), 3), 'operating_system': docker_host_values[2], }, 'measurement_note': ( 'Successful HTTP throughput, not committed-write throughput. ' 'Load generator is closed-loop. Docker stats are one-second samples ' 'and can miss short CPU peaks.' ), }, 'scenarios': scenarios, 'summary': [summarize(scenario) for scenario in scenarios], 'raw_runs': True, } summary = [summarize(scenario) for scenario in scenarios] rps_chart = output_dir / 'successful-rps.svg' p95_chart = output_dir / 'p95-latency.svg' write_chart(rps_chart, summary, 'successful_rps_mean', 'Successful HTTP throughput by scale', 'Successful requests per second') write_chart(p95_chart, summary, 'p95_latency_ms_mean', 'Successful request p95 by scale', 'Milliseconds') report['summary'] = summary report['charts'] = [rps_chart.name, p95_chart.name] output.write_text( json.dumps(report, ensure_ascii=False, indent=2) + '\n', encoding='utf-8', ) print(json.dumps(summary, ensure_ascii=False, indent=2)) print('Saved repeatable benchmark evidence to ' + str(output)) if __name__ == '__main__': try: main() except (OSError, RuntimeError, TimeoutError, ValueError, json.JSONDecodeError) as error: print('ERROR: ' + str(error), file=sys.stderr) raise SystemExit(1)