Files

102 lines
6.0 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env python3
"""Generate standalone figures and CSV/Markdown tables from raw research JSON."""
from collections import defaultdict
import csv
import json
import os
import re
from pathlib import Path
import statistics
os.environ.setdefault('MPLCONFIGDIR', '/tmp/mtusi-lab3-matplotlib')
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
ROOT = Path(__file__).resolve().parents[1]
DIRECTORY = ROOT / 'evidence/scaling'
def memory_mib(value):
number, unit = re.fullmatch(r"([0-9.]+)([A-Za-z]+)", value.split(" / ")[0]).groups()
return float(number) * {"B":1/1048576,"KiB":1/1024,"MiB":1,"GiB":1024,"kB":1000/1048576,"MB":1000000/1048576,"GB":1000000000/1048576}[unit]
def main():
groups = defaultdict(list)
for path in DIRECTORY.glob('*.json'):
data = json.loads(path.read_text())
if 'case' in data:
groups[data['case']['name']].append(data)
rows = []
for name, values in sorted(groups.items()):
rps = [d['benchmark']['successful_rps'] for d in values]
row = {'case': name, 'repeats':len(values), 'replicas':values[0]['case']['replicas'],
'rps_mean':round(statistics.mean(rps),3), 'rps_min':min(rps), 'rps_max':max(rps),
'rps_stdev':round(statistics.stdev(rps),3) if len(rps)>1 else 0,
'p50_ms_mean':round(statistics.mean(d['benchmark']['successful_latency_ms']['p50'] for d in values),3),
'p95_ms_mean':round(statistics.mean(d['benchmark']['successful_latency_ms']['p95'] for d in values),3),
'p99_ms_mean':round(statistics.mean(d['benchmark']['successful_latency_ms']['p99'] for d in values),3),
'errors':sum(sum(d['benchmark']['errors'].values()) for d in values),
'db_cpu_percent_mean':round(statistics.mean(d['cpu_percent_mean']['mtusi-lab3-db-1'] for d in values),3),
'lb_cpu_percent_mean':round(statistics.mean(d['cpu_percent_mean']['mtusi-lab3-lb-1'] for d in values),3),
'client_cpu_percent_mean':round(statistics.mean(d['cpu_percent_mean'].get('mtusi-lab3-research-load',0) for d in values),3),
'api_memory_peak_mib':round(max(memory_mib(r['MemUsage']) for d in values for r in d['stats_samples'] if r['Name'].startswith('mtusi-lab3-api-')),3) if any(r['Name'].startswith('mtusi-lab3-api-') for d in values for r in d['stats_samples']) else None,
'db_memory_peak_mib':round(max(memory_mib(r['MemUsage']) for d in values for r in d['stats_samples'] if r['Name']=='mtusi-lab3-db-1'),3),
'cache_new_connections_mean':round(statistics.mean(d['dependency_after']['valkey']['total_connections_received']-d['dependency_before']['valkey']['total_connections_received'] for d in values),3)}
rows.append(row)
with (DIRECTORY/'summary.csv').open('w') as handle:
writer = csv.DictWriter(handle,fieldnames=list(rows[0]))
writer.writeheader()
writer.writerows(rows)
(DIRECTORY/'summary.json').write_text(json.dumps(rows,indent=2)+'\n')
by_name={row['case']:row for row in rows}
plt.rcParams.update({'font.family':'DejaVu Sans','font.size':10,'axes.grid':True,'grid.alpha':.25})
fig,axes=plt.subplots(1,2,figsize=(11,4),layout='constrained')
for family,label in [('fixed','Fixed API: 1 CPU / 1024 MiB total'),('growing','Growing API: 0.5 CPU / 256 MiB each')]:
data=[by_name[f'{family}-{n}'] for n in (1,2,4) if f'{family}-{n}' in by_name]
if data:
for ax,metric in zip(axes,('rps_mean','p95_ms_mean')):
ax.plot([d['replicas'] for d in data],[d[metric] for d in data],marker='o',label=label)
axes[0].errorbar([d['replicas'] for d in data],[d['rps_mean'] for d in data],yerr=[d['rps_stdev'] for d in data],fmt='none',capsize=4)
for ax in axes:
ax.set_xlabel('API replicas')
ax.set_xticks([1,2,4])
axes[0].set_ylabel('Successful POST/s (mean ± sample SD)')
axes[1].set_ylabel('Successful request p95, ms (mean across runs)')
axes[0].legend(fontsize=8)
fig.suptitle('Replica scaling, concurrency 16; PostgreSQL limited to 0.25 CPU')
fig.savefig(DIRECTORY/'scaling.png',dpi=180)
fig.savefig(DIRECTORY/'scaling.svg')
plt.close(fig)
if all(f'saturation-c{c}' in by_name for c in (1,4,16,64)):
fig,axes=plt.subplots(1,2,figsize=(11,4),layout='constrained')
data=[by_name[f'saturation-c{c}'] for c in (1,4,16,64)]
axes[0].plot([1,4,16,64],[d['rps_mean'] for d in data],marker='o')
axes[0].set(xscale='log',xlabel='Concurrency',ylabel='Successful POST/s',title='Saturation at fixed resources')
ax2=axes[0].twinx()
ax2.plot([1,4,16,64],[d['p95_ms_mean'] for d in data],marker='s',color='tab:red')
ax2.set_ylabel('p95 ms',color='tab:red')
names=['saturation-c16','db-0.5','db-1']
if all(n in by_name for n in names):
axes[1].bar(['0.25 CPU','0.5 CPU','1 CPU'],[by_name[n]['rps_mean'] for n in names])
axes[1].set(xlabel='PostgreSQL CPU quota',ylabel='Successful POST/s',title='Bottleneck intervention (4 API, c=16)')
fig.savefig(DIRECTORY/'saturation.png',dpi=180)
fig.savefig(DIRECTORY/'saturation.svg')
plt.close(fig)
if all('cache-'+mode in by_name for mode in ('per_request','shared')):
fig,axes=plt.subplots(1,2,figsize=(10,4),layout='constrained')
names=['cache-per_request','cache-shared']
axes[0].bar(['Per-request client','Shared pool'],[by_name[n]['rps_mean'] for n in names], yerr=[by_name[n]['rps_stdev'] for n in names], capsize=5)
axes[0].set_ylabel('Successful POST/s')
axes[1].bar(['Per-request client','Shared pool'],[by_name[n]['cache_new_connections_mean'] for n in names])
axes[1].set_ylabel('New Valkey connections per full run')
fig.suptitle('Connection reuse: 2 API × 0.5 CPU, PostgreSQL 1 CPU, c=16')
fig.savefig(DIRECTORY/'cache-improvement.png',dpi=180)
fig.savefig(DIRECTORY/'cache-improvement.svg')
plt.close(fig)
print(json.dumps(rows,indent=2))
if __name__=='__main__':
main()