import argparse import csv from datetime import datetime, timezone import json import os from pathlib import Path import platform import subprocess import sys import time from concurrent.futures import ThreadPoolExecutor ROOT = Path(__file__).resolve().parents[1] OUT = ROOT / 'evidence' / 'scaling' def command(args, env=None): result = subprocess.run(args, cwd=ROOT, env=env, text=True, encoding='utf-8', capture_output=True) if result.returncode: raise RuntimeError('Command failed: ' + ' '.join(args) + '\n' + result.stderr[-2000:] + result.stdout[-1000:]) return result.stdout def deploy(replicas, cpu, memory, pool): env = dict(os.environ, API_REPLICAS=str(replicas), API_CPUS=str(cpu), API_MEMORY=str(memory) + 'm', DB_POOL_SIZE=str(pool)) command(['docker', 'compose', 'up', '-d', '--no-deps', '--scale', 'api=' + str(replicas), 'api'], env) deadline = time.monotonic() + 90 while time.monotonic() < deadline: ids = command(['docker', 'compose', 'ps', '-q', 'api'], env).split() items = json.loads(command(['docker', 'inspect', *ids], env)) if ids else [] if len(items) == replicas and all(item['State'].get('Health', {}).get('Status') == 'healthy' for item in items): break time.sleep(1) else: raise RuntimeError('API replicas did not become healthy within 90 seconds') time.sleep(4) def stats(): rows = command(['docker', 'stats', '--no-stream', '--format', '{{json .}}']).splitlines() return [json.loads(row) for row in rows if row.strip() and 'mtusi-lab3-' in json.loads(row).get('Name', '')] def dataset(): return json.loads(command(['docker', 'compose', 'exec', '-T', 'postgres', 'psql', '-U', 'messenger', '-d', 'messenger', '-tAc', "SELECT json_build_object('users',(SELECT count(*) FROM users),'chats',(SELECT count(*) FROM chats),'messages',(SELECT count(*) FROM messages),'database_bytes',pg_database_size(current_database()))"])) def benchmark(name, duration, concurrency, active=False): if not active: command([sys.executable, 'tests/load.py', '--duration', '3', '--concurrency', str(concurrency)]) started = datetime.now(timezone.utc).isoformat() snapshots = [] with ThreadPoolExecutor(max_workers=1) as executor: pending = executor.submit(command, [sys.executable, 'scripts/load_active.py' if active else 'tests/load.py', '--duration', str(duration), '--concurrency', str(concurrency), '--output', str(OUT / (name + '.json'))]) while not pending.done(): snapshots.append({'at': time.time(), 'containers': stats()}) time.sleep(0.5) pending.result() (OUT / (name + '-resources.json')).write_text(json.dumps(snapshots, indent=2), encoding='utf-8') logs = command(['docker', 'compose', 'logs', '--no-color', '--since', started, 'api']) metrics = [] for line in logs.splitlines(): if '"event": "pool_metrics"' in line: metrics.append(json.loads(line[line.index('{'):])) (OUT / (name + '-pool.json')).write_text(json.dumps(metrics, indent=2), encoding='utf-8') return json.loads((OUT / (name + '.json')).read_text(encoding='utf-8')) def graph(rows): import matplotlib matplotlib.use('Agg') import matplotlib.pyplot as plt import statistics fig, axes = plt.subplots(1, 2, figsize=(11, 4), layout='constrained') for series, color in {'fixed': '#2563eb', 'growing': '#16a34a'}.items(): groups = [[r['successful_rps'] for r in rows if r['series'] == series and r['replicas'] == count] for count in (1, 2, 4)] axes[0].errorbar([1, 2, 4], [statistics.mean(g) for g in groups], yerr=[(max(g)-min(g))/2 for g in groups], marker='o', capsize=4, color=color, label=series) axes[0].set(title='API scaling, 24 clients', xlabel='API replicas', ylabel='Successful requests / second', xticks=[1, 2, 4]) axes[0].legend() for series, label in [('saturation', 'pool=8'), ('pool-saturation', 'pool=1')]: selected = [r for r in rows if r['series'] == series] axes[1].plot([r['concurrency'] for r in selected], [r['successful_rps'] for r in selected], marker='o', label=label) axes[1].set(title='Public driver, including client startup', xlabel='Concurrent clients', ylabel='Successful requests / second') axes[1].legend() for axis in axes: axis.grid(alpha=0.25) axis.set_ylim(bottom=0) fig.savefig(OUT / 'throughput.svg') fig.savefig(OUT / 'throughput.png', dpi=160) plt.close(fig) def main(): import matplotlib parser = argparse.ArgumentParser() parser.add_argument('--duration', type=int, default=15) parser.add_argument('--repeats', type=int, default=2) args = parser.parse_args() OUT.mkdir(parents=True, exist_ok=True) metadata = {'date_utc': datetime.now(timezone.utc).isoformat(), 'matplotlib_version': matplotlib.__version__, 'base_url': os.environ.get('BASE_URL', 'http://localhost:8080'), 'platform': platform.platform(), 'logical_cpus': os.cpu_count(), 'docker': json.loads(command(['docker', 'info', '--format', '{{json .}}'])), 'duration': args.duration, 'warmup_seconds': 3, 'concurrency': 24, 'repeats': args.repeats, 'dataset_before': dataset()} safe = {key: metadata['docker'].get(key) for key in ('ServerVersion', 'NCPU', 'MemTotal', 'OperatingSystem', 'KernelVersion', 'Architecture')} metadata['docker'] = safe (OUT / 'environment.json').write_text(json.dumps(metadata, indent=2), encoding='utf-8') rows = [] try: for series in ('fixed', 'growing'): for replicas in (1, 2, 4): cpu = 1 / replicas if series == 'fixed' else 0.5 memory = 512 // replicas if series == 'fixed' else 256 deploy(replicas, cpu, memory, 8) for repeat in range(args.repeats): name = f'{series}-{replicas}-r{repeat+1}' print('Measuring ' + name, flush=True) result = benchmark(name, args.duration, 24) rows.append(dict(series=series, replicas=replicas, cpu_each=cpu, memory_mb_each=memory, pool=8, repeat=repeat+1, **result)) (OUT / 'results.json').write_text(json.dumps(rows, indent=2), encoding='utf-8') deploy(2, 0.5, 256, 1) for repeat in range(args.repeats): result = benchmark('pool-one-r' + str(repeat+1), args.duration, 24) rows.append(dict(series='pool-one', replicas=2, cpu_each=0.5, memory_mb_each=256, pool=1, repeat=repeat+1, **result)) deploy(4, 0.5, 256, 8) for concurrency in (4, 16, 64): result = benchmark('saturation-c' + str(concurrency), args.duration, concurrency) rows.append(dict(series='saturation', replicas=4, cpu_each=0.5, memory_mb_each=256, pool=8, repeat=1, **result)) deploy(4, 0.5, 256, 1) for concurrency in (4, 16, 64): result = benchmark('pool-saturation-c' + str(concurrency), args.duration, concurrency) rows.append(dict(series='pool-saturation', replicas=4, cpu_each=0.5, memory_mb_each=256, pool=1, repeat=1, **result)) (OUT / 'results.json').write_text(json.dumps(rows, indent=2), encoding='utf-8') metadata['dataset_after'] = dataset() (OUT / 'environment.json').write_text(json.dumps(metadata, indent=2), encoding='utf-8') graph(rows) fields = ['series', 'replicas', 'cpu_each', 'memory_mb_each', 'pool', 'repeat', 'concurrency', 'successful_rps', 'failed_requests', 'error_ratio'] with (OUT / 'results.csv').open('w', encoding='utf-8', newline='') as stream: writer = csv.DictWriter(stream, fieldnames=fields, extrasaction='ignore') writer.writeheader() writer.writerows(rows) finally: deploy(2, 0.5, 256, 1) if __name__ == '__main__': main()