forked from mtusi2026/mtusi2026lab3
Compare API and PostgreSQL CPU budgets and connection pool limits. Collect repeatable measurements and generate report aggregates and graphs.
221 lines
14 KiB
Python
221 lines
14 KiB
Python
#!/usr/bin/env python3
|
|
"""Keep report evidence in one JSON, one CSV, two figures and checks.json."""
|
|
import argparse
|
|
from collections import Counter
|
|
import csv
|
|
from datetime import datetime, timezone
|
|
import hashlib
|
|
import json
|
|
from pathlib import Path
|
|
import statistics
|
|
import subprocess
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
SERIES = [
|
|
('api-cpu', None, 'api-small-growth-20261029'),
|
|
('pg-cpu', 'pg0.5', 'diagnostics-v3-linux-20261028'),
|
|
('pg-cpu', 'pg1', 'diagnostics-pg1cpu-linux-20261028'),
|
|
('pool-limit', 'pool60', 'pool15-four-r100-short-20261028'),
|
|
('pool-limit', 'pool32', 'pool8-four-r100-short-20261028'),
|
|
('pg-upper-cpu', 'pg1', 'api-small-db-control-20261029/pg1'),
|
|
('pg-upper-cpu', 'pg2', 'api-small-db-control-20261029/pg2'),
|
|
]
|
|
|
|
|
|
def digest(path):
|
|
return hashlib.sha256(path.read_bytes()).hexdigest()
|
|
|
|
|
|
def flatten(bundle):
|
|
rows = []
|
|
for run in bundle['runs']:
|
|
profile, measurement = run['profile'], run['measurement']
|
|
all_requests = measurement['all']
|
|
diagnostics = run['diagnostics']
|
|
resources, pools = diagnostics['resources'], diagnostics['pools']
|
|
manifest = bundle['manifests'][run['manifest']]
|
|
containers = manifest['containers'][run['configuration']]
|
|
apis = [c for c in containers if '-api-' in c['name']]
|
|
pg = next(c for c in containers if '-postgres-' in c['name'])
|
|
seconds, rate = profile['duration_seconds'], profile['offered_rps']
|
|
acquired = sum(p['acquisitions'] for p in pools.values())
|
|
fraction = all_requests['attempted'] / (seconds * rate)
|
|
latency = all_requests['successful_latency_ms']
|
|
all_latency = run.get('all_latency_ms') or {}
|
|
rows.append({
|
|
'dataset': run['dataset'], 'variant': run['variant'], 'label': run['label'],
|
|
'replicas': len(apis), 'api_cpu': sum(c['nano_cpus'] for c in apis) / 1e9,
|
|
'api_ram_mib': sum(c['memory_bytes'] for c in apis) / 1024**2,
|
|
'pool_total': sum(int(c['pool_max']) for c in apis),
|
|
'pg_cpu': pg['nano_cpus'] / 1e9, 'pg_ram_mib': pg['memory_bytes'] / 1024**2,
|
|
'offered_rps': rate, 'duration_seconds': seconds,
|
|
'warmup_seconds': profile.get('warmup_seconds', manifest['warmup_seconds']),
|
|
'attempted': all_requests['attempted'], 'successful': all_requests['successful'],
|
|
'errors': all_requests['errors'], 'successful_rps': all_requests['successful'] / seconds,
|
|
'attempted_rps': all_requests['attempted'] / seconds,
|
|
'delivery_fraction': fraction, 'delivery_valid': .98 <= fraction <= 1.02,
|
|
'p50_ms': latency['p50'], 'p95_ms': latency['p95'], 'p99_ms': latency['p99'],
|
|
'all_request_p95_ms': all_latency.get('p95'),
|
|
'api_cpu_cores': sum(r.get('mean_cpu_cores', 0) for name, r in resources.items() if name.startswith('api-')),
|
|
'api_peak_memory_mib_sum': sum(r.get('peak_memory_mib', 0) for name, r in resources.items() if name.startswith('api-')),
|
|
'api_max_throttle_percent': max((p.get('throttled_period_fraction', 0) for name, p in resources.items() if name.startswith('api-')), default=0) * 100,
|
|
'pg_cpu_cores': resources['postgres'].get('mean_cpu_cores'),
|
|
'pg_peak_memory_mib': resources['postgres'].get('peak_memory_mib'),
|
|
'pg_throttle_percent': resources['postgres'].get('throttled_period_fraction', 0) * 100,
|
|
'host_cpu_pressure_percent': diagnostics['host'].get('cpu_some_fraction', 0) * 100,
|
|
'acquisition_mean_ms': sum(p['mean_acquisition_ms'] * p['acquisitions'] for p in pools.values()) / max(acquired, 1),
|
|
'acquisition_errors': sum(p['acquisition_errors'] for p in pools.values()),
|
|
'pool_max_waiting': max((p['max_waiting'] for p in pools.values()), default=0),
|
|
'nginx_min_ready': diagnostics['nginx']['min_ready'],
|
|
'diagnostics_complete': diagnostics['diagnostics_complete'],
|
|
})
|
|
return rows
|
|
|
|
|
|
def collect(source, output):
|
|
manifests, runs = {}, []
|
|
for dataset, variant, directory in SERIES:
|
|
path = source / directory
|
|
manifest = json.loads((path / 'manifest.json').read_text(encoding='utf-8'))
|
|
manifests[directory] = manifest
|
|
for label in manifest['runs']:
|
|
location = path / label
|
|
benchmark = json.loads((location / 'benchmark.json').read_text(encoding='utf-8'))
|
|
summary = json.loads((location / 'summary.json').read_text(encoding='utf-8'))
|
|
configuration = next(p for p in manifest['profiles'] if label.startswith(p + '-r'))
|
|
runs.append({
|
|
'dataset': dataset, 'variant': variant or configuration, 'configuration': configuration,
|
|
'manifest': directory, 'label': label,
|
|
'profile': benchmark['profile'], 'measurement': benchmark['measurement'],
|
|
'timing': benchmark.get('timing'), 'client': benchmark.get('client'),
|
|
'all_latency_ms': benchmark.get('all_latency_ms'),
|
|
'criterion': summary.get('criterion'),
|
|
'diagnostics': {key: summary[key] for key in
|
|
('resources', 'pools', 'host', 'postgres', 'nginx', 'limits_reached', 'diagnostics_complete')},
|
|
'source_hashes': {'benchmark': digest(location / 'benchmark.json'),
|
|
'summary': digest(location / 'summary.json')},
|
|
})
|
|
sources = sorted([p for p in (ROOT / 'src').rglob('*.ts') if 'generated' not in p.parts]
|
|
+ list((ROOT / 'prisma').rglob('*.sql'))
|
|
+ list((ROOT / 'configs').glob('*.yaml'))
|
|
+ [ROOT / p for p in ('compose.yaml', 'Dockerfile', 'infra/nginx.conf', 'infra/start-nginx.sh', 'pnpm-lock.yaml')])
|
|
host_path = source / 'diagnostics-pg1cpu-full-20261028/host-environment.json'
|
|
host = json.loads(host_path.read_text(encoding='utf-8'))
|
|
bundle = {
|
|
'schema_version': 1, 'recorded_at': datetime.now(timezone.utc).isoformat(),
|
|
'git_base': subprocess.check_output(['git', 'rev-parse', 'HEAD'], cwd=ROOT, text=True).strip(),
|
|
'revision_note': 'Implementation is in the working tree; git_base identifies the scaffold, not a committed implementation.',
|
|
'retained_data': 'Per-run HTTP aggregates, profiles, CPU/RAM/pool/SQL-state diagnostics, original manifests and source hashes. Request samples, stdout/stderr, traces and duplicate documentation omitted.',
|
|
'hardware': {'processor': host['processor'], 'windows_total_visible_kib': host['windows_memory_kib']['total_visible']},
|
|
'application_source_sha256': {p.relative_to(ROOT).as_posix(): digest(p) for p in sources},
|
|
'manifests': manifests, 'runs': runs,
|
|
}
|
|
(output / 'measurements.json').write_text(json.dumps(bundle, ensure_ascii=False, indent=2) + '\n', encoding='utf-8')
|
|
return bundle
|
|
|
|
|
|
def verify(bundle, rows):
|
|
counts = Counter(r['dataset'] for r in rows)
|
|
assert counts == {'api-cpu': 36, 'pg-cpu': 6, 'pool-limit': 6, 'pg-upper-cpu': 6}, counts
|
|
assert len({(r['dataset'], r['variant'], r['label']) for r in rows}) == 54
|
|
assert all(r['attempted'] == r['successful'] + r['errors'] for r in rows)
|
|
assert all(r['diagnostics_complete'] for r in rows)
|
|
matrix = [r for r in rows if r['dataset'] == 'api-cpu']
|
|
assert all(r['pool_total'] == 32 and r['api_ram_mib'] == 1024 and r['pg_cpu'] == 1 for r in matrix)
|
|
assert all(n == 3 for n in Counter((r['variant'], r['offered_rps']) for r in matrix).values())
|
|
for name, expected in [('pg-cpu', {'pg0.5': .5, 'pg1': 1}),
|
|
('pg-upper-cpu', {'pg1': 1, 'pg2': 2})]:
|
|
assert all(r['pg_cpu'] == expected[r['variant']] for r in rows if r['dataset'] == name)
|
|
assert {r['pool_total'] for r in rows if r['dataset'] == 'pool-limit'} == {32, 60}
|
|
assert len(bundle['manifests']) == 7
|
|
|
|
|
|
def plot(rows, output):
|
|
import matplotlib
|
|
matplotlib.use('Agg')
|
|
import matplotlib.pyplot as plt
|
|
|
|
matrix = [r for r in rows if r['dataset'] == 'api-cpu']
|
|
fig, axes = plt.subplots(2, 2, figsize=(12, 8))
|
|
for cpu, color in ((1, '#9333ea'), (2, '#059669')):
|
|
for ax, rate, field, title, ylabel in (
|
|
(axes[0, 0], 40, 'all_request_p95_ms', '40 scheduled RPS: all-request latency', 'Median p95, ms'),
|
|
(axes[0, 1], 80, 'successful_rps', '80 scheduled RPS: successful throughput', 'Median successful requests/s'),
|
|
(axes[1, 0], 40, 'api_max_throttle_percent', '40 RPS: most throttled API', 'Median throttled periods, %'),
|
|
(axes[1, 1], 80, 'delivery_fraction', '80 RPS: actual load delivery', 'Median started / scheduled, %'),
|
|
):
|
|
groups = [[r for r in matrix if r['api_cpu'] == cpu and r['replicas'] == n and r['offered_rps'] == rate] for n in (1, 2, 4)]
|
|
scale = 100 if field == 'delivery_fraction' else 1
|
|
values = [statistics.median(r[field] for r in g) * scale for g in groups]
|
|
ax.plot([1, 2, 4], values, marker='o', color=color, label=f'API total {cpu} CPU / 1 GiB')
|
|
ax.fill_between([1, 2, 4], [min(r[field] for r in g) * scale for g in groups],
|
|
[max(r[field] for r in g) * scale for g in groups], color=color, alpha=.1)
|
|
ax.set(title=title, xlabel='API replicas', ylabel=ylabel, xticks=[1, 2, 4])
|
|
ax.grid(alpha=.25)
|
|
ax.legend(fontsize=8)
|
|
axes[0, 0].axhline(1000, color='#64748b', linestyle='--')
|
|
axes[1, 1].axhline(98, color='#64748b', linestyle='--')
|
|
fig.suptitle('Fixed budgets and CPU growth; PG 1 CPU / 512 MiB; total pool 32', fontsize=13)
|
|
fig.text(.5, .012, 'All repeats shown, including underfed runs. Lines: medians; bands: min/max, not confidence intervals. No measured plateau claim.', ha='center', fontsize=8)
|
|
fig.tight_layout(rect=(0, .04, 1, .95))
|
|
fig.savefig(output / 'scaling.png', dpi=160)
|
|
plt.close(fig)
|
|
|
|
fig, axes = plt.subplots(2, 2, figsize=(12, 8))
|
|
panels = [
|
|
('api-cpu', ['two-small', 'two'], 'all_request_p95_ms', ['API 1 CPU', 'API 2 CPU'], '2 API, 40 RPS: CPU increase', 'All-request p95, ms', 40),
|
|
('pg-cpu', ['pg0.5', 'pg1'], 'p95_ms', ['PG 0.5 CPU', 'PG 1 CPU'], '4 API, 60 RPS: database quota', 'Successful-request p95, ms', 60),
|
|
('pool-limit', ['pool60', 'pool32'], 'successful_rps', ['60 connections', '32 connections'], '4 API, 100 scheduled RPS: pool limit', 'Successful requests/s', 100),
|
|
('pg-upper-cpu', ['pg1', 'pg2'], 'all_request_p95_ms', ['PG 1 CPU', 'PG 2 CPU'], '2 API, 80 RPS: further PG CPU increase', 'All-request p95, ms', 80),
|
|
]
|
|
for ax, (dataset, variants, field, labels, title, ylabel, rate) in zip(axes.flat, panels):
|
|
for i, variant in enumerate(variants):
|
|
group = [r for r in rows if r['dataset'] == dataset and r['variant'] == variant and r['offered_rps'] == rate]
|
|
assert len(group) == 3
|
|
color = '#9333ea' if i == 0 else '#059669'
|
|
for j, r in enumerate(group):
|
|
# Older Node series use a different scheduler: show their observations without a retroactive k6 classification.
|
|
underfed = dataset in {'api-cpu', 'pg-upper-cpu'} and not r['delivery_valid']
|
|
ax.scatter(i + (j - 1) * .08, r[field], color=color, marker='x' if underfed else 'o', s=55)
|
|
value = (sum(r['successful'] for r in group) / sum(r['duration_seconds'] for r in group)
|
|
if dataset == 'pool-limit' else statistics.median(r[field] for r in group))
|
|
ax.hlines(value, i - .17, i + .17, color='#0f172a', linewidth=2)
|
|
ax.annotate(f'{value:.0f}' if value > 100 else f'{value:.2f}', (i, value),
|
|
xytext=(12, 8), textcoords='offset points', fontsize=9)
|
|
ax.set(title=title, ylabel=ylabel, xticks=[0, 1], xticklabels=labels, xlim=(-.4, 1.4))
|
|
ax.grid(axis='y', alpha=.25)
|
|
if field == 'all_request_p95_ms':
|
|
ax.axhline(1000, color='#64748b', linestyle='--', linewidth=1)
|
|
maximum = max(r[field] for r in rows if r['dataset'] == dataset and r['variant'] in variants and r['offered_rps'] == rate)
|
|
ax.set_ylim(0, maximum * 1.2)
|
|
fig.suptitle('Separate experiments: compare variants within each panel', fontsize=13)
|
|
fig.text(.5, .012, 'Dots: individual runs; black bars: medians (pool panel: aggregate successful RPS). Crosses: k6 underdelivery. See REPORT.md for scheduler differences.', ha='center', fontsize=8)
|
|
fig.tight_layout(rect=(0, .04, 1, .95))
|
|
fig.savefig(output / 'improvements.png', dpi=160)
|
|
plt.close(fig)
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument('--collect', action='store_true', help='Import the selected raw experiment directories before cleanup')
|
|
parser.add_argument('--source-root', type=Path, default=ROOT / '.artifacts/scaling')
|
|
parser.add_argument('--output', type=Path, default=ROOT / 'evidence')
|
|
parser.add_argument('--input', type=Path, default=ROOT / 'evidence/measurements.json')
|
|
parser.add_argument('--plot', action='store_true')
|
|
args = parser.parse_args()
|
|
args.output.mkdir(parents=True, exist_ok=True)
|
|
bundle = collect(args.source_root, args.output) if args.collect else json.loads(args.input.read_text(encoding='utf-8'))
|
|
rows = flatten(bundle)
|
|
verify(bundle, rows)
|
|
with (args.output / 'scaling.csv').open('w', encoding='utf-8', newline='') as stream:
|
|
writer = csv.DictWriter(stream, fieldnames=rows[0].keys())
|
|
writer.writeheader()
|
|
writer.writerows(rows)
|
|
if args.plot:
|
|
plot(rows, args.output)
|
|
print(f'Verified {len(rows)} runs from {len(bundle["manifests"])} source series; compact evidence in {args.output}')
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|