@@ -0,0 +1,124 @@
|
||||
"""Build the track 1 summary and figures from recorded JSON/CSV, without Docker."""
|
||||
from collections import defaultdict
|
||||
import csv
|
||||
from datetime import datetime, timedelta
|
||||
import json
|
||||
from pathlib import Path
|
||||
from statistics import mean, median
|
||||
|
||||
import matplotlib
|
||||
matplotlib.use('Agg')
|
||||
import matplotlib.pyplot as plt
|
||||
|
||||
DATA = Path(__file__).resolve().parents[1] / 'evidence' / 'track1'
|
||||
|
||||
|
||||
def summarize(path):
|
||||
run = json.loads(path.read_text())
|
||||
resources = json.loads(path.with_name(path.stem + '-resources.json').read_text())
|
||||
case = resources['case']
|
||||
start = datetime.fromisoformat(run['workload_started_at'])
|
||||
first = start + timedelta(seconds=run['warmup_seconds']+2)
|
||||
last = start + timedelta(seconds=run['warmup_seconds']+run['requested_seconds'])
|
||||
cores = defaultdict(list)
|
||||
for sample in resources['resource_samples']:
|
||||
if first <= datetime.fromisoformat(sample['at']) <= last:
|
||||
totals = defaultdict(float)
|
||||
for container in sample['containers']:
|
||||
service = container['Name'].removeprefix('mtusi2026lab3-track1-').split('-')[0]
|
||||
totals[service] += float(container['CPUPerc'].rstrip('%'))/100
|
||||
for service, value in totals.items():
|
||||
cores[service].append(value)
|
||||
active = locked = connections = samples = 0
|
||||
for sample in run['db_samples']:
|
||||
if run['warmup_seconds'] <= sample['second'] <= run['warmup_seconds']+run['requested_seconds']:
|
||||
samples += 1
|
||||
for state in sample['states']:
|
||||
connections += state['connections']
|
||||
if state['state'] == 'active':
|
||||
active += state['connections']
|
||||
if state['wait_type'] == 'Lock':
|
||||
locked += state['connections']
|
||||
requests = run['successes'] + run['failed_requests']
|
||||
row = {'name': run['name'], 'replicas': case['replicas'], 'api_cpu_total': case['cpu']*case['replicas'],
|
||||
'api_ram_total_mib': int(case['memory'].rstrip('m'))*case['replicas'],
|
||||
'concurrency': run['concurrency'], 'rps': run['successful_rps'],
|
||||
'errors': run['failed_requests'], 'error_percent': round(100*run['failed_requests']/requests, 3),
|
||||
**run['latency_ms'], 'db_lock_percent_active': round(100*locked/active, 2) if active else 0,
|
||||
'db_connections_mean': round(connections/samples, 2),
|
||||
**{service+'_cpu_cores_mean': round(mean(cores[service]), 3) if cores[service] else None
|
||||
for service in ['api', 'nginx', 'postgres', 'redis', 'bench']}}
|
||||
# Verify aggregates against the request-level source data.
|
||||
with path.with_name(path.stem+'-requests.csv').open(newline='') as stream:
|
||||
raw = list(csv.DictReader(stream))
|
||||
assert len(raw) == requests
|
||||
assert sum(bool(r['error']) for r in raw) == row['errors']
|
||||
assert run['confirmed_messages_preserved']
|
||||
return row
|
||||
|
||||
|
||||
def interval(rows, field):
|
||||
values = [row[field] for row in rows]
|
||||
mid = median(values)
|
||||
return mid, mid-min(values), max(values)-mid
|
||||
|
||||
|
||||
def main():
|
||||
paths = [p for p in sorted(DATA.glob('*.json'))
|
||||
if p.stem.startswith(('fixed-', 'growing-', 'saturation-', 'improvement-'))
|
||||
and not p.stem.endswith('-resources')]
|
||||
rows = [summarize(path) for path in paths]
|
||||
with (DATA / 'summary.csv').open('w', newline='') as stream:
|
||||
writer = csv.DictWriter(stream, fieldnames=list(rows[0]))
|
||||
writer.writeheader()
|
||||
writer.writerows(rows)
|
||||
plt.rcParams.update({'font.size': 11, 'axes.grid': True, 'grid.alpha': .25})
|
||||
fig, axes = plt.subplots(1, 2, figsize=(11, 4), layout='constrained')
|
||||
for series, label in [('fixed', 'API budget: 2 CPU / 1024 MiB'), ('growing', 'Each API: 1 CPU / 512 MiB')]:
|
||||
for axis, field, ylabel in zip(axes, ['rps', 'p95'], ['Successful messages/s', 'p95 latency, ms']):
|
||||
groups = [[r for r in rows if r['name'].startswith(f'{series}-n{n}-')] for n in [1, 2, 4]]
|
||||
if not all(groups):
|
||||
continue
|
||||
values = [interval(group, field) for group in groups]
|
||||
axis.errorbar([1, 2, 4], [v[0] for v in values],
|
||||
yerr=[[v[1] for v in values], [v[2] for v in values]],
|
||||
marker='o', capsize=5, label=label)
|
||||
axis.set(xlabel='API replicas', ylabel=ylabel, xticks=[1, 2, 4])
|
||||
axis.legend(fontsize=9)
|
||||
fig.suptitle('Same workload: 8 clients, one chat; median and min–max of 3 runs')
|
||||
fig.savefig(DATA / 'scaling.png', dpi=160)
|
||||
plt.close(fig)
|
||||
|
||||
saturation = sorted([r for r in rows if r['name'].startswith('saturation-')], key=lambda r: r['concurrency'])
|
||||
fig, axes = plt.subplots(1, 3, figsize=(13, 4), layout='constrained')
|
||||
for axis, field, ylabel in zip(axes, ['rps', 'p95', 'db_lock_percent_active'],
|
||||
['Successful messages/s', 'p95 latency, ms', 'Active DB samples waiting for locks, %']):
|
||||
axis.plot([r['concurrency'] for r in saturation], [r[field] for r in saturation], marker='o')
|
||||
axis.set(xlabel='Concurrent clients', ylabel=ylabel, xscale='log', xticks=[1, 4, 8, 16, 32, 64])
|
||||
axis.set_xticklabels([1, 4, 8, 16, 32, 64])
|
||||
for row in saturation:
|
||||
if row['errors']:
|
||||
axes[0].annotate(f"{row['error_percent']:.1f}% errors", (row['concurrency'], row['rps']),
|
||||
xytext=(-4, 10), textcoords='offset points', ha='right', fontsize=8)
|
||||
fig.suptitle('Saturation: 4 API × 1 CPU; one run per load level')
|
||||
fig.savefig(DATA / 'saturation.png', dpi=160)
|
||||
plt.close(fig)
|
||||
|
||||
groups = [[r for r in rows if r['name'].startswith(f'improvement-{v}-')] for v in ['before', 'after']]
|
||||
if all(groups):
|
||||
fig, axes = plt.subplots(1, 2, figsize=(9, 4), layout='constrained')
|
||||
for axis, field, ylabel in zip(axes, ['rps', 'p95'], ['Successful messages/s', 'p95 latency, ms']):
|
||||
values = [interval(group, field) for group in groups]
|
||||
axis.bar(['Before', 'After'], [v[0] for v in values], color=['#627d98', '#159c80'],
|
||||
yerr=[[v[1] for v in values], [v[2] for v in values]], capsize=5)
|
||||
axis.set(ylabel=ylabel)
|
||||
for index, value in enumerate(values):
|
||||
axis.text(index, value[0]/2, f'{value[0]:.1f}', ha='center', color='white', fontsize=14)
|
||||
fig.suptitle('4 × 0.5 CPU → 2 × 1 CPU; same total budget, 3 pairs × 45 s')
|
||||
fig.savefig(DATA / 'improvement.png', dpi=160)
|
||||
plt.close(fig)
|
||||
print(f'Wrote summary.csv and figures for {len(rows)} runs')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Reference in New Issue
Block a user