Files

144 lines
7.7 KiB
Python

import argparse
import csv
from datetime import datetime, timezone
import json
import os
from pathlib import Path
import platform
import subprocess
import sys
import time
from concurrent.futures import ThreadPoolExecutor
ROOT = Path(__file__).resolve().parents[1]
OUT = ROOT / 'evidence' / 'scaling'
def command(args, env=None):
result = subprocess.run(args, cwd=ROOT, env=env, text=True, encoding='utf-8', capture_output=True)
if result.returncode:
raise RuntimeError('Command failed: ' + ' '.join(args) + '\n' + result.stderr[-2000:] + result.stdout[-1000:])
return result.stdout
def deploy(replicas, cpu, memory, pool):
env = dict(os.environ, API_REPLICAS=str(replicas), API_CPUS=str(cpu), API_MEMORY=str(memory) + 'm', DB_POOL_SIZE=str(pool))
command(['docker', 'compose', 'up', '-d', '--no-deps', '--scale', 'api=' + str(replicas), 'api'], env)
deadline = time.monotonic() + 90
while time.monotonic() < deadline:
ids = command(['docker', 'compose', 'ps', '-q', 'api'], env).split()
items = json.loads(command(['docker', 'inspect', *ids], env)) if ids else []
if len(items) == replicas and all(item['State'].get('Health', {}).get('Status') == 'healthy' for item in items):
break
time.sleep(1)
else:
raise RuntimeError('API replicas did not become healthy within 90 seconds')
time.sleep(4)
def stats():
rows = command(['docker', 'stats', '--no-stream', '--format', '{{json .}}']).splitlines()
return [json.loads(row) for row in rows if row.strip() and 'mtusi-lab3-' in json.loads(row).get('Name', '')]
def dataset():
return json.loads(command(['docker', 'compose', 'exec', '-T', 'postgres', 'psql', '-U', 'messenger', '-d', 'messenger', '-tAc', "SELECT json_build_object('users',(SELECT count(*) FROM users),'chats',(SELECT count(*) FROM chats),'messages',(SELECT count(*) FROM messages),'database_bytes',pg_database_size(current_database()))"]))
def benchmark(name, duration, concurrency, active=False):
if not active:
command([sys.executable, 'tests/load.py', '--duration', '3', '--concurrency', str(concurrency)])
started = datetime.now(timezone.utc).isoformat()
snapshots = []
with ThreadPoolExecutor(max_workers=1) as executor:
pending = executor.submit(command, [sys.executable, 'scripts/load_active.py' if active else 'tests/load.py', '--duration', str(duration), '--concurrency', str(concurrency), '--output', str(OUT / (name + '.json'))])
while not pending.done():
snapshots.append({'at': time.time(), 'containers': stats()})
time.sleep(0.5)
pending.result()
(OUT / (name + '-resources.json')).write_text(json.dumps(snapshots, indent=2), encoding='utf-8')
logs = command(['docker', 'compose', 'logs', '--no-color', '--since', started, 'api'])
metrics = []
for line in logs.splitlines():
if '"event": "pool_metrics"' in line:
metrics.append(json.loads(line[line.index('{'):]))
(OUT / (name + '-pool.json')).write_text(json.dumps(metrics, indent=2), encoding='utf-8')
return json.loads((OUT / (name + '.json')).read_text(encoding='utf-8'))
def graph(rows):
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
import statistics
fig, axes = plt.subplots(1, 2, figsize=(11, 4), layout='constrained')
for series, color in {'fixed': '#2563eb', 'growing': '#16a34a'}.items():
groups = [[r['successful_rps'] for r in rows if r['series'] == series and r['replicas'] == count] for count in (1, 2, 4)]
axes[0].errorbar([1, 2, 4], [statistics.mean(g) for g in groups], yerr=[(max(g)-min(g))/2 for g in groups], marker='o', capsize=4, color=color, label=series)
axes[0].set(title='API scaling, 24 clients', xlabel='API replicas', ylabel='Successful requests / second', xticks=[1, 2, 4])
axes[0].legend()
for series, label in [('saturation', 'pool=8'), ('pool-saturation', 'pool=1')]:
selected = [r for r in rows if r['series'] == series]
axes[1].plot([r['concurrency'] for r in selected], [r['successful_rps'] for r in selected], marker='o', label=label)
axes[1].set(title='Public driver, including client startup', xlabel='Concurrent clients', ylabel='Successful requests / second')
axes[1].legend()
for axis in axes:
axis.grid(alpha=0.25)
axis.set_ylim(bottom=0)
fig.savefig(OUT / 'throughput.svg')
fig.savefig(OUT / 'throughput.png', dpi=160)
plt.close(fig)
def main():
import matplotlib
parser = argparse.ArgumentParser()
parser.add_argument('--duration', type=int, default=15)
parser.add_argument('--repeats', type=int, default=2)
args = parser.parse_args()
OUT.mkdir(parents=True, exist_ok=True)
metadata = {'date_utc': datetime.now(timezone.utc).isoformat(), 'matplotlib_version': matplotlib.__version__, 'base_url': os.environ.get('BASE_URL', 'http://localhost:8080'), 'platform': platform.platform(), 'logical_cpus': os.cpu_count(), 'docker': json.loads(command(['docker', 'info', '--format', '{{json .}}'])), 'duration': args.duration, 'warmup_seconds': 3, 'concurrency': 24, 'repeats': args.repeats, 'dataset_before': dataset()}
safe = {key: metadata['docker'].get(key) for key in ('ServerVersion', 'NCPU', 'MemTotal', 'OperatingSystem', 'KernelVersion', 'Architecture')}
metadata['docker'] = safe
(OUT / 'environment.json').write_text(json.dumps(metadata, indent=2), encoding='utf-8')
rows = []
try:
for series in ('fixed', 'growing'):
for replicas in (1, 2, 4):
cpu = 1 / replicas if series == 'fixed' else 0.5
memory = 512 // replicas if series == 'fixed' else 256
deploy(replicas, cpu, memory, 8)
for repeat in range(args.repeats):
name = f'{series}-{replicas}-r{repeat+1}'
print('Measuring ' + name, flush=True)
result = benchmark(name, args.duration, 24)
rows.append(dict(series=series, replicas=replicas, cpu_each=cpu, memory_mb_each=memory, pool=8, repeat=repeat+1, **result))
(OUT / 'results.json').write_text(json.dumps(rows, indent=2), encoding='utf-8')
deploy(2, 0.5, 256, 1)
for repeat in range(args.repeats):
result = benchmark('pool-one-r' + str(repeat+1), args.duration, 24)
rows.append(dict(series='pool-one', replicas=2, cpu_each=0.5, memory_mb_each=256, pool=1, repeat=repeat+1, **result))
deploy(4, 0.5, 256, 8)
for concurrency in (4, 16, 64):
result = benchmark('saturation-c' + str(concurrency), args.duration, concurrency)
rows.append(dict(series='saturation', replicas=4, cpu_each=0.5, memory_mb_each=256, pool=8, repeat=1, **result))
deploy(4, 0.5, 256, 1)
for concurrency in (4, 16, 64):
result = benchmark('pool-saturation-c' + str(concurrency), args.duration, concurrency)
rows.append(dict(series='pool-saturation', replicas=4, cpu_each=0.5, memory_mb_each=256, pool=1, repeat=1, **result))
(OUT / 'results.json').write_text(json.dumps(rows, indent=2), encoding='utf-8')
metadata['dataset_after'] = dataset()
(OUT / 'environment.json').write_text(json.dumps(metadata, indent=2), encoding='utf-8')
graph(rows)
fields = ['series', 'replicas', 'cpu_each', 'memory_mb_each', 'pool', 'repeat', 'concurrency', 'successful_rps', 'failed_requests', 'error_ratio']
with (OUT / 'results.csv').open('w', encoding='utf-8', newline='') as stream:
writer = csv.DictWriter(stream, fieldnames=fields, extrasaction='ignore')
writer.writeheader()
writer.writerows(rows)
finally:
deploy(2, 0.5, 256, 1)
if __name__ == '__main__':
main()