forked from mtusi2026/mtusi2026lab3
102 lines
6.0 KiB
Python
102 lines
6.0 KiB
Python
#!/usr/bin/env python3
|
||
"""Generate standalone figures and CSV/Markdown tables from raw research JSON."""
|
||
from collections import defaultdict
|
||
import csv
|
||
import json
|
||
import os
|
||
import re
|
||
from pathlib import Path
|
||
import statistics
|
||
|
||
os.environ.setdefault('MPLCONFIGDIR', '/tmp/mtusi-lab3-matplotlib')
|
||
import matplotlib
|
||
matplotlib.use('Agg')
|
||
import matplotlib.pyplot as plt
|
||
|
||
ROOT = Path(__file__).resolve().parents[1]
|
||
DIRECTORY = ROOT / 'evidence/scaling'
|
||
|
||
|
||
def memory_mib(value):
|
||
number, unit = re.fullmatch(r"([0-9.]+)([A-Za-z]+)", value.split(" / ")[0]).groups()
|
||
return float(number) * {"B":1/1048576,"KiB":1/1024,"MiB":1,"GiB":1024,"kB":1000/1048576,"MB":1000000/1048576,"GB":1000000000/1048576}[unit]
|
||
|
||
|
||
def main():
|
||
groups = defaultdict(list)
|
||
for path in DIRECTORY.glob('*.json'):
|
||
data = json.loads(path.read_text())
|
||
if 'case' in data:
|
||
groups[data['case']['name']].append(data)
|
||
rows = []
|
||
for name, values in sorted(groups.items()):
|
||
rps = [d['benchmark']['successful_rps'] for d in values]
|
||
row = {'case': name, 'repeats':len(values), 'replicas':values[0]['case']['replicas'],
|
||
'rps_mean':round(statistics.mean(rps),3), 'rps_min':min(rps), 'rps_max':max(rps),
|
||
'rps_stdev':round(statistics.stdev(rps),3) if len(rps)>1 else 0,
|
||
'p50_ms_mean':round(statistics.mean(d['benchmark']['successful_latency_ms']['p50'] for d in values),3),
|
||
'p95_ms_mean':round(statistics.mean(d['benchmark']['successful_latency_ms']['p95'] for d in values),3),
|
||
'p99_ms_mean':round(statistics.mean(d['benchmark']['successful_latency_ms']['p99'] for d in values),3),
|
||
'errors':sum(sum(d['benchmark']['errors'].values()) for d in values),
|
||
'db_cpu_percent_mean':round(statistics.mean(d['cpu_percent_mean']['mtusi-lab3-db-1'] for d in values),3),
|
||
'lb_cpu_percent_mean':round(statistics.mean(d['cpu_percent_mean']['mtusi-lab3-lb-1'] for d in values),3),
|
||
'client_cpu_percent_mean':round(statistics.mean(d['cpu_percent_mean'].get('mtusi-lab3-research-load',0) for d in values),3),
|
||
'api_memory_peak_mib':round(max(memory_mib(r['MemUsage']) for d in values for r in d['stats_samples'] if r['Name'].startswith('mtusi-lab3-api-')),3) if any(r['Name'].startswith('mtusi-lab3-api-') for d in values for r in d['stats_samples']) else None,
|
||
'db_memory_peak_mib':round(max(memory_mib(r['MemUsage']) for d in values for r in d['stats_samples'] if r['Name']=='mtusi-lab3-db-1'),3),
|
||
'cache_new_connections_mean':round(statistics.mean(d['dependency_after']['valkey']['total_connections_received']-d['dependency_before']['valkey']['total_connections_received'] for d in values),3)}
|
||
rows.append(row)
|
||
with (DIRECTORY/'summary.csv').open('w') as handle:
|
||
writer = csv.DictWriter(handle,fieldnames=list(rows[0]))
|
||
writer.writeheader()
|
||
writer.writerows(rows)
|
||
(DIRECTORY/'summary.json').write_text(json.dumps(rows,indent=2)+'\n')
|
||
by_name={row['case']:row for row in rows}
|
||
plt.rcParams.update({'font.family':'DejaVu Sans','font.size':10,'axes.grid':True,'grid.alpha':.25})
|
||
fig,axes=plt.subplots(1,2,figsize=(11,4),layout='constrained')
|
||
for family,label in [('fixed','Fixed API: 1 CPU / 1024 MiB total'),('growing','Growing API: 0.5 CPU / 256 MiB each')]:
|
||
data=[by_name[f'{family}-{n}'] for n in (1,2,4) if f'{family}-{n}' in by_name]
|
||
if data:
|
||
for ax,metric in zip(axes,('rps_mean','p95_ms_mean')):
|
||
ax.plot([d['replicas'] for d in data],[d[metric] for d in data],marker='o',label=label)
|
||
axes[0].errorbar([d['replicas'] for d in data],[d['rps_mean'] for d in data],yerr=[d['rps_stdev'] for d in data],fmt='none',capsize=4)
|
||
for ax in axes:
|
||
ax.set_xlabel('API replicas')
|
||
ax.set_xticks([1,2,4])
|
||
axes[0].set_ylabel('Successful POST/s (mean ± sample SD)')
|
||
axes[1].set_ylabel('Successful request p95, ms (mean across runs)')
|
||
axes[0].legend(fontsize=8)
|
||
fig.suptitle('Replica scaling, concurrency 16; PostgreSQL limited to 0.25 CPU')
|
||
fig.savefig(DIRECTORY/'scaling.png',dpi=180)
|
||
fig.savefig(DIRECTORY/'scaling.svg')
|
||
plt.close(fig)
|
||
if all(f'saturation-c{c}' in by_name for c in (1,4,16,64)):
|
||
fig,axes=plt.subplots(1,2,figsize=(11,4),layout='constrained')
|
||
data=[by_name[f'saturation-c{c}'] for c in (1,4,16,64)]
|
||
axes[0].plot([1,4,16,64],[d['rps_mean'] for d in data],marker='o')
|
||
axes[0].set(xscale='log',xlabel='Concurrency',ylabel='Successful POST/s',title='Saturation at fixed resources')
|
||
ax2=axes[0].twinx()
|
||
ax2.plot([1,4,16,64],[d['p95_ms_mean'] for d in data],marker='s',color='tab:red')
|
||
ax2.set_ylabel('p95 ms',color='tab:red')
|
||
names=['saturation-c16','db-0.5','db-1']
|
||
if all(n in by_name for n in names):
|
||
axes[1].bar(['0.25 CPU','0.5 CPU','1 CPU'],[by_name[n]['rps_mean'] for n in names])
|
||
axes[1].set(xlabel='PostgreSQL CPU quota',ylabel='Successful POST/s',title='Bottleneck intervention (4 API, c=16)')
|
||
fig.savefig(DIRECTORY/'saturation.png',dpi=180)
|
||
fig.savefig(DIRECTORY/'saturation.svg')
|
||
plt.close(fig)
|
||
if all('cache-'+mode in by_name for mode in ('per_request','shared')):
|
||
fig,axes=plt.subplots(1,2,figsize=(10,4),layout='constrained')
|
||
names=['cache-per_request','cache-shared']
|
||
axes[0].bar(['Per-request client','Shared pool'],[by_name[n]['rps_mean'] for n in names], yerr=[by_name[n]['rps_stdev'] for n in names], capsize=5)
|
||
axes[0].set_ylabel('Successful POST/s')
|
||
axes[1].bar(['Per-request client','Shared pool'],[by_name[n]['cache_new_connections_mean'] for n in names])
|
||
axes[1].set_ylabel('New Valkey connections per full run')
|
||
fig.suptitle('Connection reuse: 2 API × 0.5 CPU, PostgreSQL 1 CPU, c=16')
|
||
fig.savefig(DIRECTORY/'cache-improvement.png',dpi=180)
|
||
fig.savefig(DIRECTORY/'cache-improvement.svg')
|
||
plt.close(fig)
|
||
print(json.dumps(rows,indent=2))
|
||
|
||
if __name__=='__main__':
|
||
main()
|