forked from mtusi2026/mtusi2026lab3
144 lines
7.7 KiB
Python
144 lines
7.7 KiB
Python
import argparse
|
|
import csv
|
|
from datetime import datetime, timezone
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import platform
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
OUT = ROOT / 'evidence' / 'scaling'
|
|
|
|
|
|
def command(args, env=None):
|
|
result = subprocess.run(args, cwd=ROOT, env=env, text=True, encoding='utf-8', capture_output=True)
|
|
if result.returncode:
|
|
raise RuntimeError('Command failed: ' + ' '.join(args) + '\n' + result.stderr[-2000:] + result.stdout[-1000:])
|
|
return result.stdout
|
|
|
|
|
|
def deploy(replicas, cpu, memory, pool):
|
|
env = dict(os.environ, API_REPLICAS=str(replicas), API_CPUS=str(cpu), API_MEMORY=str(memory) + 'm', DB_POOL_SIZE=str(pool))
|
|
command(['docker', 'compose', 'up', '-d', '--no-deps', '--scale', 'api=' + str(replicas), 'api'], env)
|
|
deadline = time.monotonic() + 90
|
|
while time.monotonic() < deadline:
|
|
ids = command(['docker', 'compose', 'ps', '-q', 'api'], env).split()
|
|
items = json.loads(command(['docker', 'inspect', *ids], env)) if ids else []
|
|
if len(items) == replicas and all(item['State'].get('Health', {}).get('Status') == 'healthy' for item in items):
|
|
break
|
|
time.sleep(1)
|
|
else:
|
|
raise RuntimeError('API replicas did not become healthy within 90 seconds')
|
|
time.sleep(4)
|
|
|
|
|
|
def stats():
|
|
rows = command(['docker', 'stats', '--no-stream', '--format', '{{json .}}']).splitlines()
|
|
return [json.loads(row) for row in rows if row.strip() and 'mtusi-lab3-' in json.loads(row).get('Name', '')]
|
|
|
|
|
|
def dataset():
|
|
return json.loads(command(['docker', 'compose', 'exec', '-T', 'postgres', 'psql', '-U', 'messenger', '-d', 'messenger', '-tAc', "SELECT json_build_object('users',(SELECT count(*) FROM users),'chats',(SELECT count(*) FROM chats),'messages',(SELECT count(*) FROM messages),'database_bytes',pg_database_size(current_database()))"]))
|
|
|
|
|
|
def benchmark(name, duration, concurrency, active=False):
|
|
if not active:
|
|
command([sys.executable, 'tests/load.py', '--duration', '3', '--concurrency', str(concurrency)])
|
|
started = datetime.now(timezone.utc).isoformat()
|
|
snapshots = []
|
|
with ThreadPoolExecutor(max_workers=1) as executor:
|
|
pending = executor.submit(command, [sys.executable, 'scripts/load_active.py' if active else 'tests/load.py', '--duration', str(duration), '--concurrency', str(concurrency), '--output', str(OUT / (name + '.json'))])
|
|
while not pending.done():
|
|
snapshots.append({'at': time.time(), 'containers': stats()})
|
|
time.sleep(0.5)
|
|
pending.result()
|
|
(OUT / (name + '-resources.json')).write_text(json.dumps(snapshots, indent=2), encoding='utf-8')
|
|
logs = command(['docker', 'compose', 'logs', '--no-color', '--since', started, 'api'])
|
|
metrics = []
|
|
for line in logs.splitlines():
|
|
if '"event": "pool_metrics"' in line:
|
|
metrics.append(json.loads(line[line.index('{'):]))
|
|
(OUT / (name + '-pool.json')).write_text(json.dumps(metrics, indent=2), encoding='utf-8')
|
|
return json.loads((OUT / (name + '.json')).read_text(encoding='utf-8'))
|
|
|
|
|
|
def graph(rows):
|
|
import matplotlib
|
|
matplotlib.use('Agg')
|
|
import matplotlib.pyplot as plt
|
|
import statistics
|
|
fig, axes = plt.subplots(1, 2, figsize=(11, 4), layout='constrained')
|
|
for series, color in {'fixed': '#2563eb', 'growing': '#16a34a'}.items():
|
|
groups = [[r['successful_rps'] for r in rows if r['series'] == series and r['replicas'] == count] for count in (1, 2, 4)]
|
|
axes[0].errorbar([1, 2, 4], [statistics.mean(g) for g in groups], yerr=[(max(g)-min(g))/2 for g in groups], marker='o', capsize=4, color=color, label=series)
|
|
axes[0].set(title='API scaling, 24 clients', xlabel='API replicas', ylabel='Successful requests / second', xticks=[1, 2, 4])
|
|
axes[0].legend()
|
|
for series, label in [('saturation', 'pool=8'), ('pool-saturation', 'pool=1')]:
|
|
selected = [r for r in rows if r['series'] == series]
|
|
axes[1].plot([r['concurrency'] for r in selected], [r['successful_rps'] for r in selected], marker='o', label=label)
|
|
axes[1].set(title='Public driver, including client startup', xlabel='Concurrent clients', ylabel='Successful requests / second')
|
|
axes[1].legend()
|
|
for axis in axes:
|
|
axis.grid(alpha=0.25)
|
|
axis.set_ylim(bottom=0)
|
|
fig.savefig(OUT / 'throughput.svg')
|
|
fig.savefig(OUT / 'throughput.png', dpi=160)
|
|
plt.close(fig)
|
|
|
|
|
|
def main():
|
|
import matplotlib
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument('--duration', type=int, default=15)
|
|
parser.add_argument('--repeats', type=int, default=2)
|
|
args = parser.parse_args()
|
|
OUT.mkdir(parents=True, exist_ok=True)
|
|
metadata = {'date_utc': datetime.now(timezone.utc).isoformat(), 'matplotlib_version': matplotlib.__version__, 'base_url': os.environ.get('BASE_URL', 'http://localhost:8080'), 'platform': platform.platform(), 'logical_cpus': os.cpu_count(), 'docker': json.loads(command(['docker', 'info', '--format', '{{json .}}'])), 'duration': args.duration, 'warmup_seconds': 3, 'concurrency': 24, 'repeats': args.repeats, 'dataset_before': dataset()}
|
|
safe = {key: metadata['docker'].get(key) for key in ('ServerVersion', 'NCPU', 'MemTotal', 'OperatingSystem', 'KernelVersion', 'Architecture')}
|
|
metadata['docker'] = safe
|
|
(OUT / 'environment.json').write_text(json.dumps(metadata, indent=2), encoding='utf-8')
|
|
rows = []
|
|
try:
|
|
for series in ('fixed', 'growing'):
|
|
for replicas in (1, 2, 4):
|
|
cpu = 1 / replicas if series == 'fixed' else 0.5
|
|
memory = 512 // replicas if series == 'fixed' else 256
|
|
deploy(replicas, cpu, memory, 8)
|
|
for repeat in range(args.repeats):
|
|
name = f'{series}-{replicas}-r{repeat+1}'
|
|
print('Measuring ' + name, flush=True)
|
|
result = benchmark(name, args.duration, 24)
|
|
rows.append(dict(series=series, replicas=replicas, cpu_each=cpu, memory_mb_each=memory, pool=8, repeat=repeat+1, **result))
|
|
(OUT / 'results.json').write_text(json.dumps(rows, indent=2), encoding='utf-8')
|
|
deploy(2, 0.5, 256, 1)
|
|
for repeat in range(args.repeats):
|
|
result = benchmark('pool-one-r' + str(repeat+1), args.duration, 24)
|
|
rows.append(dict(series='pool-one', replicas=2, cpu_each=0.5, memory_mb_each=256, pool=1, repeat=repeat+1, **result))
|
|
deploy(4, 0.5, 256, 8)
|
|
for concurrency in (4, 16, 64):
|
|
result = benchmark('saturation-c' + str(concurrency), args.duration, concurrency)
|
|
rows.append(dict(series='saturation', replicas=4, cpu_each=0.5, memory_mb_each=256, pool=8, repeat=1, **result))
|
|
deploy(4, 0.5, 256, 1)
|
|
for concurrency in (4, 16, 64):
|
|
result = benchmark('pool-saturation-c' + str(concurrency), args.duration, concurrency)
|
|
rows.append(dict(series='pool-saturation', replicas=4, cpu_each=0.5, memory_mb_each=256, pool=1, repeat=1, **result))
|
|
(OUT / 'results.json').write_text(json.dumps(rows, indent=2), encoding='utf-8')
|
|
metadata['dataset_after'] = dataset()
|
|
(OUT / 'environment.json').write_text(json.dumps(metadata, indent=2), encoding='utf-8')
|
|
graph(rows)
|
|
fields = ['series', 'replicas', 'cpu_each', 'memory_mb_each', 'pool', 'repeat', 'concurrency', 'successful_rps', 'failed_requests', 'error_ratio']
|
|
with (OUT / 'results.csv').open('w', encoding='utf-8', newline='') as stream:
|
|
writer = csv.DictWriter(stream, fieldnames=fields, extrasaction='ignore')
|
|
writer.writeheader()
|
|
writer.writerows(rows)
|
|
finally:
|
|
deploy(2, 0.5, 256, 1)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|