125 lines
6.5 KiB
Python
125 lines
6.5 KiB
Python
"""Build the track 1 summary and figures from recorded JSON/CSV, without Docker."""
|
||
from collections import defaultdict
|
||
import csv
|
||
from datetime import datetime, timedelta
|
||
import json
|
||
from pathlib import Path
|
||
from statistics import mean, median
|
||
|
||
import matplotlib
|
||
matplotlib.use('Agg')
|
||
import matplotlib.pyplot as plt
|
||
|
||
DATA = Path(__file__).resolve().parents[1] / 'evidence' / 'track1'
|
||
|
||
|
||
def summarize(path):
|
||
run = json.loads(path.read_text())
|
||
resources = json.loads(path.with_name(path.stem + '-resources.json').read_text())
|
||
case = resources['case']
|
||
start = datetime.fromisoformat(run['workload_started_at'])
|
||
first = start + timedelta(seconds=run['warmup_seconds']+2)
|
||
last = start + timedelta(seconds=run['warmup_seconds']+run['requested_seconds'])
|
||
cores = defaultdict(list)
|
||
for sample in resources['resource_samples']:
|
||
if first <= datetime.fromisoformat(sample['at']) <= last:
|
||
totals = defaultdict(float)
|
||
for container in sample['containers']:
|
||
service = container['Name'].removeprefix('mtusi2026lab3-track1-').split('-')[0]
|
||
totals[service] += float(container['CPUPerc'].rstrip('%'))/100
|
||
for service, value in totals.items():
|
||
cores[service].append(value)
|
||
active = locked = connections = samples = 0
|
||
for sample in run['db_samples']:
|
||
if run['warmup_seconds'] <= sample['second'] <= run['warmup_seconds']+run['requested_seconds']:
|
||
samples += 1
|
||
for state in sample['states']:
|
||
connections += state['connections']
|
||
if state['state'] == 'active':
|
||
active += state['connections']
|
||
if state['wait_type'] == 'Lock':
|
||
locked += state['connections']
|
||
requests = run['successes'] + run['failed_requests']
|
||
row = {'name': run['name'], 'replicas': case['replicas'], 'api_cpu_total': case['cpu']*case['replicas'],
|
||
'api_ram_total_mib': int(case['memory'].rstrip('m'))*case['replicas'],
|
||
'concurrency': run['concurrency'], 'rps': run['successful_rps'],
|
||
'errors': run['failed_requests'], 'error_percent': round(100*run['failed_requests']/requests, 3),
|
||
**run['latency_ms'], 'db_lock_percent_active': round(100*locked/active, 2) if active else 0,
|
||
'db_connections_mean': round(connections/samples, 2),
|
||
**{service+'_cpu_cores_mean': round(mean(cores[service]), 3) if cores[service] else None
|
||
for service in ['api', 'nginx', 'postgres', 'redis', 'bench']}}
|
||
# Verify aggregates against the request-level source data.
|
||
with path.with_name(path.stem+'-requests.csv').open(newline='') as stream:
|
||
raw = list(csv.DictReader(stream))
|
||
assert len(raw) == requests
|
||
assert sum(bool(r['error']) for r in raw) == row['errors']
|
||
assert run['confirmed_messages_preserved']
|
||
return row
|
||
|
||
|
||
def interval(rows, field):
|
||
values = [row[field] for row in rows]
|
||
mid = median(values)
|
||
return mid, mid-min(values), max(values)-mid
|
||
|
||
|
||
def main():
|
||
paths = [p for p in sorted(DATA.glob('*.json'))
|
||
if p.stem.startswith(('fixed-', 'growing-', 'saturation-', 'improvement-'))
|
||
and not p.stem.endswith('-resources')]
|
||
rows = [summarize(path) for path in paths]
|
||
with (DATA / 'summary.csv').open('w', newline='') as stream:
|
||
writer = csv.DictWriter(stream, fieldnames=list(rows[0]))
|
||
writer.writeheader()
|
||
writer.writerows(rows)
|
||
plt.rcParams.update({'font.size': 11, 'axes.grid': True, 'grid.alpha': .25})
|
||
fig, axes = plt.subplots(1, 2, figsize=(11, 4), layout='constrained')
|
||
for series, label in [('fixed', 'API budget: 2 CPU / 1024 MiB'), ('growing', 'Each API: 1 CPU / 512 MiB')]:
|
||
for axis, field, ylabel in zip(axes, ['rps', 'p95'], ['Successful messages/s', 'p95 latency, ms']):
|
||
groups = [[r for r in rows if r['name'].startswith(f'{series}-n{n}-')] for n in [1, 2, 4]]
|
||
if not all(groups):
|
||
continue
|
||
values = [interval(group, field) for group in groups]
|
||
axis.errorbar([1, 2, 4], [v[0] for v in values],
|
||
yerr=[[v[1] for v in values], [v[2] for v in values]],
|
||
marker='o', capsize=5, label=label)
|
||
axis.set(xlabel='API replicas', ylabel=ylabel, xticks=[1, 2, 4])
|
||
axis.legend(fontsize=9)
|
||
fig.suptitle('Same workload: 8 clients, one chat; median and min–max of 3 runs')
|
||
fig.savefig(DATA / 'scaling.png', dpi=160)
|
||
plt.close(fig)
|
||
|
||
saturation = sorted([r for r in rows if r['name'].startswith('saturation-')], key=lambda r: r['concurrency'])
|
||
fig, axes = plt.subplots(1, 3, figsize=(13, 4), layout='constrained')
|
||
for axis, field, ylabel in zip(axes, ['rps', 'p95', 'db_lock_percent_active'],
|
||
['Successful messages/s', 'p95 latency, ms', 'Active DB samples waiting for locks, %']):
|
||
axis.plot([r['concurrency'] for r in saturation], [r[field] for r in saturation], marker='o')
|
||
axis.set(xlabel='Concurrent clients', ylabel=ylabel, xscale='log', xticks=[1, 4, 8, 16, 32, 64])
|
||
axis.set_xticklabels([1, 4, 8, 16, 32, 64])
|
||
for row in saturation:
|
||
if row['errors']:
|
||
axes[0].annotate(f"{row['error_percent']:.1f}% errors", (row['concurrency'], row['rps']),
|
||
xytext=(-4, 10), textcoords='offset points', ha='right', fontsize=8)
|
||
fig.suptitle('Saturation: 4 API × 1 CPU; one run per load level')
|
||
fig.savefig(DATA / 'saturation.png', dpi=160)
|
||
plt.close(fig)
|
||
|
||
groups = [[r for r in rows if r['name'].startswith(f'improvement-{v}-')] for v in ['before', 'after']]
|
||
if all(groups):
|
||
fig, axes = plt.subplots(1, 2, figsize=(9, 4), layout='constrained')
|
||
for axis, field, ylabel in zip(axes, ['rps', 'p95'], ['Successful messages/s', 'p95 latency, ms']):
|
||
values = [interval(group, field) for group in groups]
|
||
axis.bar(['Before', 'After'], [v[0] for v in values], color=['#627d98', '#159c80'],
|
||
yerr=[[v[1] for v in values], [v[2] for v in values]], capsize=5)
|
||
axis.set(ylabel=ylabel)
|
||
for index, value in enumerate(values):
|
||
axis.text(index, value[0]/2, f'{value[0]:.1f}', ha='center', color='white', fontsize=14)
|
||
fig.suptitle('4 × 0.5 CPU → 2 × 1 CPU; same total budget, 3 pairs × 45 s')
|
||
fig.savefig(DATA / 'improvement.png', dpi=160)
|
||
plt.close(fig)
|
||
print(f'Wrote summary.csv and figures for {len(rows)} runs')
|
||
|
||
|
||
if __name__ == '__main__':
|
||
main()
|