forked from mtusi2026/mtusi2026lab3
131 lines
7.6 KiB
Python
131 lines
7.6 KiB
Python
#!/usr/bin/env python3
|
|
"""Rebuild static research figures and aggregate metrics from raw summaries."""
|
|
from collections import defaultdict
|
|
import argparse
|
|
import json
|
|
from pathlib import Path
|
|
import statistics
|
|
|
|
import matplotlib
|
|
matplotlib.use('Agg')
|
|
import matplotlib.pyplot as plt
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument('--directory', default='evidence/scaling')
|
|
args = parser.parse_args()
|
|
dest = ROOT / args.directory
|
|
source_caption = 'Source: ' + args.directory + '/summary.json; 2026-11-06'
|
|
records = json.loads((dest / 'summary.json').read_text(encoding='utf-8'))
|
|
grouped = defaultdict(list)
|
|
for record in records:
|
|
grouped[(record['series'], record['replicas'], record['concurrency'])].append(record)
|
|
aggregate = []
|
|
for (series, replicas, concurrency), runs in grouped.items():
|
|
cpu, ram = defaultdict(list), defaultdict(list)
|
|
for run in runs:
|
|
resource = json.loads((dest / (run['name'] + '.resources.json')).read_text(encoding='utf-8'))
|
|
for sample in resource['resource_samples']:
|
|
for c in sample['containers']:
|
|
service = next((s for s in ('api', 'db', 'lb', 'tester') if '-' + s + '-' in c['Name']), None)
|
|
if not service:
|
|
continue
|
|
cpu[service].append(float(c['CPUPerc'].rstrip('%')))
|
|
value = c['MemUsage'].split('/')[0].strip()
|
|
for unit, factor in [('KiB', 1 / 1024), ('MiB', 1), ('GiB', 1024), ('B', 1 / 1048576)]:
|
|
if value.endswith(unit):
|
|
ram[service].append(float(value[:-len(unit)]) * factor)
|
|
break
|
|
aggregate.append({'series': series, 'replicas': replicas, 'concurrency': concurrency,
|
|
'runs': len(runs), 'api_cpu_each': runs[0]['api_cpu_each'],
|
|
'api_memory_each': runs[0]['api_memory_each'], 'db_cpu': runs[0]['db_cpu'],
|
|
'db_pool_size': runs[0].get('db_pool_size', 4),
|
|
'median_rps': statistics.median(r['successful_rps'] for r in runs),
|
|
'min_rps': min(r['successful_rps'] for r in runs),
|
|
'max_rps': max(r['successful_rps'] for r in runs),
|
|
'failed_requests_total': sum(r['failed_requests'] for r in runs),
|
|
'attempts_total': sum(r['successful_requests'] + r['failed_requests'] for r in runs),
|
|
'median_success_latency_ms': {p: statistics.median(r['successful_latency_ms'][p] for r in runs)
|
|
for p in ('p50', 'p95', 'p99')},
|
|
'sampled_cpu_percent_one_core': {s: {'mean': round(statistics.mean(v), 2), 'max': max(v)} for s, v in cpu.items()},
|
|
'sampled_ram_mib_max_per_container': {s: round(max(v), 2) for s, v in ram.items() if v},
|
|
'resource_note': 'Samples span bootstrap/warmup/measurement; CPU 100%=one core. API samples are per replica, not summed.'})
|
|
order = {s: i for i, s in enumerate(('fixed', 'growing', 'db-improved',
|
|
'saturation-baseline', 'saturation-improved', 'pool-four', 'pool-one', 'lb-unbounded', 'lb-bounded'))}
|
|
aggregate.sort(key=lambda r: (order[r['series']], r['replicas'], r['concurrency']))
|
|
(dest / 'aggregate.json').write_text(json.dumps(aggregate, indent=2) + '\n', encoding='utf-8')
|
|
plt.rcParams.update({'figure.dpi': 150, 'axes.spines.top': False, 'axes.spines.right': False})
|
|
fig, axes = plt.subplots(1, 2, figsize=(11, 4.5), layout='constrained')
|
|
for series, label, color in [('fixed', 'Fixed total API: 1 CPU / 512 MiB', '#2563eb'),
|
|
('growing', 'Each API: 0.5 CPU / 256 MiB', '#b45309')]:
|
|
rows = sorted((r for r in aggregate if r['series'] == series), key=lambda r: r['replicas'])
|
|
x, y = [r['replicas'] for r in rows], [r['median_rps'] for r in rows]
|
|
axes[0].errorbar(x, y, yerr=[[r['median_rps'] - r['min_rps'] for r in rows],
|
|
[r['max_rps'] - r['median_rps'] for r in rows]],
|
|
marker='o', capsize=4, label=label, color=color)
|
|
axes[1].plot(x, [r['median_success_latency_ms']['p95'] for r in rows], marker='o', label=label, color=color)
|
|
for ax in axes:
|
|
ax.set_xlabel('API replicas (count)')
|
|
ax.set_xticks([1, 2, 4])
|
|
ax.grid(alpha=.2)
|
|
ax.legend(fontsize=8)
|
|
axes[0].set_ylabel('Successful HTTP 201 acknowledgements / second')
|
|
axes[0].set_title('Scaling: median RPS; whiskers=min/max across repeats')
|
|
axes[1].set_ylabel('Successful request p95 latency (ms)')
|
|
axes[1].set_title('Scaling: median of run-level p95')
|
|
fig.suptitle('POST messages via HAProxy; concurrency 32; DB 0.25 CPU\n' + source_caption, fontsize=11)
|
|
fig.savefig(dest / 'scaling.png')
|
|
plt.close(fig)
|
|
fig, axes = plt.subplots(1, 2, figsize=(11, 4.5), layout='constrained')
|
|
for series, label, color in [('saturation-baseline', 'DB 0.25 CPU', '#b45309'),
|
|
('saturation-improved', 'DB 1.0 CPU', '#2563eb')]:
|
|
rows = sorted((r for r in aggregate if r['series'] == series), key=lambda r: r['concurrency'])
|
|
x = [r['concurrency'] for r in rows]
|
|
axes[0].errorbar(x, [r['median_rps'] for r in rows],
|
|
yerr=[[r['median_rps'] - r['min_rps'] for r in rows], [r['max_rps'] - r['median_rps'] for r in rows]],
|
|
marker='o', capsize=4, label=label, color=color)
|
|
axes[1].plot(x, [r['median_success_latency_ms']['p95'] for r in rows], marker='o', label=label, color=color)
|
|
for ax in axes:
|
|
ax.set_xlabel('Concurrent closed-loop clients (count)')
|
|
ax.set_xticks([1, 4, 8, 32, 64])
|
|
ax.grid(alpha=.2)
|
|
ax.legend()
|
|
axes[0].set_ylabel('Successful HTTP 201 acknowledgements / second')
|
|
axes[0].set_title('Shared dependency saturation: median and min/max')
|
|
axes[1].set_ylabel('Successful request p95 latency (ms)')
|
|
axes[1].set_title('Cost of concurrency: median of run-level p95')
|
|
fig.suptitle('4 API x 0.5 CPU / 256 MiB; only DB CPU changes\n' + source_caption, fontsize=11)
|
|
fig.savefig(dest / 'saturation.png')
|
|
plt.close(fig)
|
|
controls = [r for r in records if r['series'] == 'pool-four']
|
|
candidates = [r for r in records if r['series'] == 'pool-one']
|
|
if controls and candidates:
|
|
controls.sort(key=lambda r: r['repeat'])
|
|
candidates.sort(key=lambda r: r['repeat'])
|
|
fig, axes = plt.subplots(1, 2, figsize=(10, 4.5), layout='constrained')
|
|
for series, label, offset, color in [(controls, 'Pool 4 per API', -.18, '#b45309'),
|
|
(candidates, 'Pool 1 per API', .18, '#2563eb')]:
|
|
x = [r['repeat'] + offset for r in series]
|
|
axes[0].bar(x, [r['successful_rps'] for r in series], width=.35, label=label, color=color)
|
|
axes[1].bar(x, [r['successful_latency_ms']['p95'] for r in series], width=.35, label=label, color=color)
|
|
for ax in axes:
|
|
ax.set_xlabel('Alternating experiment pair (number)')
|
|
ax.set_xticks([r['repeat'] for r in controls])
|
|
ax.legend()
|
|
ax.grid(axis='y', alpha=.2)
|
|
axes[0].set_ylabel('Successful HTTP 201 acknowledgements / second')
|
|
axes[0].set_title('Connection pool control: observed RPS per run')
|
|
axes[1].set_ylabel('Successful request p95 latency (ms)')
|
|
axes[1].set_title('Connection pool control: observed p95 per run')
|
|
fig.suptitle('Only pool limit changes: 4 API; DB 0.25 CPU; concurrency 32\n' + source_caption, fontsize=10)
|
|
fig.savefig(dest / 'pool.png')
|
|
plt.close(fig)
|
|
print(json.dumps(aggregate, indent=2))
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|