Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
2026-10-23 17:50:43 +03:00
co-authored by Cursor
parent 1d85938fb4
commit 3417e22ba0
102 changed files with 112776 additions and 171 deletions
+124
View File
@@ -0,0 +1,124 @@
"""Build the track 1 summary and figures from recorded JSON/CSV, without Docker."""
from collections import defaultdict
import csv
from datetime import datetime, timedelta
import json
from pathlib import Path
from statistics import mean, median
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
DATA = Path(__file__).resolve().parents[1] / 'evidence' / 'track1'
def summarize(path):
run = json.loads(path.read_text())
resources = json.loads(path.with_name(path.stem + '-resources.json').read_text())
case = resources['case']
start = datetime.fromisoformat(run['workload_started_at'])
first = start + timedelta(seconds=run['warmup_seconds']+2)
last = start + timedelta(seconds=run['warmup_seconds']+run['requested_seconds'])
cores = defaultdict(list)
for sample in resources['resource_samples']:
if first <= datetime.fromisoformat(sample['at']) <= last:
totals = defaultdict(float)
for container in sample['containers']:
service = container['Name'].removeprefix('mtusi2026lab3-track1-').split('-')[0]
totals[service] += float(container['CPUPerc'].rstrip('%'))/100
for service, value in totals.items():
cores[service].append(value)
active = locked = connections = samples = 0
for sample in run['db_samples']:
if run['warmup_seconds'] <= sample['second'] <= run['warmup_seconds']+run['requested_seconds']:
samples += 1
for state in sample['states']:
connections += state['connections']
if state['state'] == 'active':
active += state['connections']
if state['wait_type'] == 'Lock':
locked += state['connections']
requests = run['successes'] + run['failed_requests']
row = {'name': run['name'], 'replicas': case['replicas'], 'api_cpu_total': case['cpu']*case['replicas'],
'api_ram_total_mib': int(case['memory'].rstrip('m'))*case['replicas'],
'concurrency': run['concurrency'], 'rps': run['successful_rps'],
'errors': run['failed_requests'], 'error_percent': round(100*run['failed_requests']/requests, 3),
**run['latency_ms'], 'db_lock_percent_active': round(100*locked/active, 2) if active else 0,
'db_connections_mean': round(connections/samples, 2),
**{service+'_cpu_cores_mean': round(mean(cores[service]), 3) if cores[service] else None
for service in ['api', 'nginx', 'postgres', 'redis', 'bench']}}
# Verify aggregates against the request-level source data.
with path.with_name(path.stem+'-requests.csv').open(newline='') as stream:
raw = list(csv.DictReader(stream))
assert len(raw) == requests
assert sum(bool(r['error']) for r in raw) == row['errors']
assert run['confirmed_messages_preserved']
return row
def interval(rows, field):
values = [row[field] for row in rows]
mid = median(values)
return mid, mid-min(values), max(values)-mid
def main():
paths = [p for p in sorted(DATA.glob('*.json'))
if p.stem.startswith(('fixed-', 'growing-', 'saturation-', 'improvement-'))
and not p.stem.endswith('-resources')]
rows = [summarize(path) for path in paths]
with (DATA / 'summary.csv').open('w', newline='') as stream:
writer = csv.DictWriter(stream, fieldnames=list(rows[0]))
writer.writeheader()
writer.writerows(rows)
plt.rcParams.update({'font.size': 11, 'axes.grid': True, 'grid.alpha': .25})
fig, axes = plt.subplots(1, 2, figsize=(11, 4), layout='constrained')
for series, label in [('fixed', 'API budget: 2 CPU / 1024 MiB'), ('growing', 'Each API: 1 CPU / 512 MiB')]:
for axis, field, ylabel in zip(axes, ['rps', 'p95'], ['Successful messages/s', 'p95 latency, ms']):
groups = [[r for r in rows if r['name'].startswith(f'{series}-n{n}-')] for n in [1, 2, 4]]
if not all(groups):
continue
values = [interval(group, field) for group in groups]
axis.errorbar([1, 2, 4], [v[0] for v in values],
yerr=[[v[1] for v in values], [v[2] for v in values]],
marker='o', capsize=5, label=label)
axis.set(xlabel='API replicas', ylabel=ylabel, xticks=[1, 2, 4])
axis.legend(fontsize=9)
fig.suptitle('Same workload: 8 clients, one chat; median and min–max of 3 runs')
fig.savefig(DATA / 'scaling.png', dpi=160)
plt.close(fig)
saturation = sorted([r for r in rows if r['name'].startswith('saturation-')], key=lambda r: r['concurrency'])
fig, axes = plt.subplots(1, 3, figsize=(13, 4), layout='constrained')
for axis, field, ylabel in zip(axes, ['rps', 'p95', 'db_lock_percent_active'],
['Successful messages/s', 'p95 latency, ms', 'Active DB samples waiting for locks, %']):
axis.plot([r['concurrency'] for r in saturation], [r[field] for r in saturation], marker='o')
axis.set(xlabel='Concurrent clients', ylabel=ylabel, xscale='log', xticks=[1, 4, 8, 16, 32, 64])
axis.set_xticklabels([1, 4, 8, 16, 32, 64])
for row in saturation:
if row['errors']:
axes[0].annotate(f"{row['error_percent']:.1f}% errors", (row['concurrency'], row['rps']),
xytext=(-4, 10), textcoords='offset points', ha='right', fontsize=8)
fig.suptitle('Saturation: 4 API × 1 CPU; one run per load level')
fig.savefig(DATA / 'saturation.png', dpi=160)
plt.close(fig)
groups = [[r for r in rows if r['name'].startswith(f'improvement-{v}-')] for v in ['before', 'after']]
if all(groups):
fig, axes = plt.subplots(1, 2, figsize=(9, 4), layout='constrained')
for axis, field, ylabel in zip(axes, ['rps', 'p95'], ['Successful messages/s', 'p95 latency, ms']):
values = [interval(group, field) for group in groups]
axis.bar(['Before', 'After'], [v[0] for v in values], color=['#627d98', '#159c80'],
yerr=[[v[1] for v in values], [v[2] for v in values]], capsize=5)
axis.set(ylabel=ylabel)
for index, value in enumerate(values):
axis.text(index, value[0]/2, f'{value[0]:.1f}', ha='center', color='white', fontsize=14)
fig.suptitle('4 × 0.5 CPU → 2 × 1 CPU; same total budget, 3 pairs × 45 s')
fig.savefig(DATA / 'improvement.png', dpi=160)
plt.close(fig)
print(f'Wrote summary.csv and figures for {len(rows)} runs')
if __name__ == '__main__':
main()