#!/usr/bin/env python3 """Keep report evidence in one JSON, one CSV, two figures and checks.json.""" import argparse from collections import Counter import csv from datetime import datetime, timezone import hashlib import json from pathlib import Path import statistics import subprocess ROOT = Path(__file__).resolve().parents[1] SERIES = [ ('api-cpu', None, 'api-small-growth-20261029'), ('pg-cpu', 'pg0.5', 'diagnostics-v3-linux-20261028'), ('pg-cpu', 'pg1', 'diagnostics-pg1cpu-linux-20261028'), ('pool-limit', 'pool60', 'pool15-four-r100-short-20261028'), ('pool-limit', 'pool32', 'pool8-four-r100-short-20261028'), ('pg-upper-cpu', 'pg1', 'api-small-db-control-20261029/pg1'), ('pg-upper-cpu', 'pg2', 'api-small-db-control-20261029/pg2'), ] def digest(path): return hashlib.sha256(path.read_bytes()).hexdigest() def flatten(bundle): rows = [] for run in bundle['runs']: profile, measurement = run['profile'], run['measurement'] all_requests = measurement['all'] diagnostics = run['diagnostics'] resources, pools = diagnostics['resources'], diagnostics['pools'] manifest = bundle['manifests'][run['manifest']] containers = manifest['containers'][run['configuration']] apis = [c for c in containers if '-api-' in c['name']] pg = next(c for c in containers if '-postgres-' in c['name']) seconds, rate = profile['duration_seconds'], profile['offered_rps'] acquired = sum(p['acquisitions'] for p in pools.values()) fraction = all_requests['attempted'] / (seconds * rate) latency = all_requests['successful_latency_ms'] all_latency = run.get('all_latency_ms') or {} rows.append({ 'dataset': run['dataset'], 'variant': run['variant'], 'label': run['label'], 'replicas': len(apis), 'api_cpu': sum(c['nano_cpus'] for c in apis) / 1e9, 'api_ram_mib': sum(c['memory_bytes'] for c in apis) / 1024**2, 'pool_total': sum(int(c['pool_max']) for c in apis), 'pg_cpu': pg['nano_cpus'] / 1e9, 'pg_ram_mib': pg['memory_bytes'] / 1024**2, 'offered_rps': rate, 'duration_seconds': seconds, 'warmup_seconds': profile.get('warmup_seconds', manifest['warmup_seconds']), 'attempted': all_requests['attempted'], 'successful': all_requests['successful'], 'errors': all_requests['errors'], 'successful_rps': all_requests['successful'] / seconds, 'attempted_rps': all_requests['attempted'] / seconds, 'delivery_fraction': fraction, 'delivery_valid': .98 <= fraction <= 1.02, 'p50_ms': latency['p50'], 'p95_ms': latency['p95'], 'p99_ms': latency['p99'], 'all_request_p95_ms': all_latency.get('p95'), 'api_cpu_cores': sum(r.get('mean_cpu_cores', 0) for name, r in resources.items() if name.startswith('api-')), 'api_peak_memory_mib_sum': sum(r.get('peak_memory_mib', 0) for name, r in resources.items() if name.startswith('api-')), 'api_max_throttle_percent': max((p.get('throttled_period_fraction', 0) for name, p in resources.items() if name.startswith('api-')), default=0) * 100, 'pg_cpu_cores': resources['postgres'].get('mean_cpu_cores'), 'pg_peak_memory_mib': resources['postgres'].get('peak_memory_mib'), 'pg_throttle_percent': resources['postgres'].get('throttled_period_fraction', 0) * 100, 'host_cpu_pressure_percent': diagnostics['host'].get('cpu_some_fraction', 0) * 100, 'acquisition_mean_ms': sum(p['mean_acquisition_ms'] * p['acquisitions'] for p in pools.values()) / max(acquired, 1), 'acquisition_errors': sum(p['acquisition_errors'] for p in pools.values()), 'pool_max_waiting': max((p['max_waiting'] for p in pools.values()), default=0), 'nginx_min_ready': diagnostics['nginx']['min_ready'], 'diagnostics_complete': diagnostics['diagnostics_complete'], }) return rows def collect(source, output): manifests, runs = {}, [] for dataset, variant, directory in SERIES: path = source / directory manifest = json.loads((path / 'manifest.json').read_text(encoding='utf-8')) manifests[directory] = manifest for label in manifest['runs']: location = path / label benchmark = json.loads((location / 'benchmark.json').read_text(encoding='utf-8')) summary = json.loads((location / 'summary.json').read_text(encoding='utf-8')) configuration = next(p for p in manifest['profiles'] if label.startswith(p + '-r')) runs.append({ 'dataset': dataset, 'variant': variant or configuration, 'configuration': configuration, 'manifest': directory, 'label': label, 'profile': benchmark['profile'], 'measurement': benchmark['measurement'], 'timing': benchmark.get('timing'), 'client': benchmark.get('client'), 'all_latency_ms': benchmark.get('all_latency_ms'), 'criterion': summary.get('criterion'), 'diagnostics': {key: summary[key] for key in ('resources', 'pools', 'host', 'postgres', 'nginx', 'limits_reached', 'diagnostics_complete')}, 'source_hashes': {'benchmark': digest(location / 'benchmark.json'), 'summary': digest(location / 'summary.json')}, }) sources = sorted([p for p in (ROOT / 'src').rglob('*.ts') if 'generated' not in p.parts] + list((ROOT / 'prisma').rglob('*.sql')) + list((ROOT / 'configs').glob('*.yaml')) + [ROOT / p for p in ('compose.yaml', 'Dockerfile', 'infra/nginx.conf', 'infra/start-nginx.sh', 'pnpm-lock.yaml')]) host_path = source / 'diagnostics-pg1cpu-full-20261028/host-environment.json' host = json.loads(host_path.read_text(encoding='utf-8')) bundle = { 'schema_version': 1, 'recorded_at': datetime.now(timezone.utc).isoformat(), 'git_base': subprocess.check_output(['git', 'rev-parse', 'HEAD'], cwd=ROOT, text=True).strip(), 'revision_note': 'Implementation is in the working tree; git_base identifies the scaffold, not a committed implementation.', 'retained_data': 'Per-run HTTP aggregates, profiles, CPU/RAM/pool/SQL-state diagnostics, original manifests and source hashes. Request samples, stdout/stderr, traces and duplicate documentation omitted.', 'hardware': {'processor': host['processor'], 'windows_total_visible_kib': host['windows_memory_kib']['total_visible']}, 'application_source_sha256': {p.relative_to(ROOT).as_posix(): digest(p) for p in sources}, 'manifests': manifests, 'runs': runs, } (output / 'measurements.json').write_text(json.dumps(bundle, ensure_ascii=False, indent=2) + '\n', encoding='utf-8') return bundle def verify(bundle, rows): counts = Counter(r['dataset'] for r in rows) assert counts == {'api-cpu': 36, 'pg-cpu': 6, 'pool-limit': 6, 'pg-upper-cpu': 6}, counts assert len({(r['dataset'], r['variant'], r['label']) for r in rows}) == 54 assert all(r['attempted'] == r['successful'] + r['errors'] for r in rows) assert all(r['diagnostics_complete'] for r in rows) matrix = [r for r in rows if r['dataset'] == 'api-cpu'] assert all(r['pool_total'] == 32 and r['api_ram_mib'] == 1024 and r['pg_cpu'] == 1 for r in matrix) assert all(n == 3 for n in Counter((r['variant'], r['offered_rps']) for r in matrix).values()) for name, expected in [('pg-cpu', {'pg0.5': .5, 'pg1': 1}), ('pg-upper-cpu', {'pg1': 1, 'pg2': 2})]: assert all(r['pg_cpu'] == expected[r['variant']] for r in rows if r['dataset'] == name) assert {r['pool_total'] for r in rows if r['dataset'] == 'pool-limit'} == {32, 60} assert len(bundle['manifests']) == 7 def plot(rows, output): import matplotlib matplotlib.use('Agg') import matplotlib.pyplot as plt matrix = [r for r in rows if r['dataset'] == 'api-cpu'] fig, axes = plt.subplots(2, 2, figsize=(12, 8)) for cpu, color in ((1, '#9333ea'), (2, '#059669')): for ax, rate, field, title, ylabel in ( (axes[0, 0], 40, 'all_request_p95_ms', '40 scheduled RPS: all-request latency', 'Median p95, ms'), (axes[0, 1], 80, 'successful_rps', '80 scheduled RPS: successful throughput', 'Median successful requests/s'), (axes[1, 0], 40, 'api_max_throttle_percent', '40 RPS: most throttled API', 'Median throttled periods, %'), (axes[1, 1], 80, 'delivery_fraction', '80 RPS: actual load delivery', 'Median started / scheduled, %'), ): groups = [[r for r in matrix if r['api_cpu'] == cpu and r['replicas'] == n and r['offered_rps'] == rate] for n in (1, 2, 4)] scale = 100 if field == 'delivery_fraction' else 1 values = [statistics.median(r[field] for r in g) * scale for g in groups] ax.plot([1, 2, 4], values, marker='o', color=color, label=f'API total {cpu} CPU / 1 GiB') ax.fill_between([1, 2, 4], [min(r[field] for r in g) * scale for g in groups], [max(r[field] for r in g) * scale for g in groups], color=color, alpha=.1) ax.set(title=title, xlabel='API replicas', ylabel=ylabel, xticks=[1, 2, 4]) ax.grid(alpha=.25) ax.legend(fontsize=8) axes[0, 0].axhline(1000, color='#64748b', linestyle='--') axes[1, 1].axhline(98, color='#64748b', linestyle='--') fig.suptitle('Fixed budgets and CPU growth; PG 1 CPU / 512 MiB; total pool 32', fontsize=13) fig.text(.5, .012, 'All repeats shown, including underfed runs. Lines: medians; bands: min/max, not confidence intervals. No measured plateau claim.', ha='center', fontsize=8) fig.tight_layout(rect=(0, .04, 1, .95)) fig.savefig(output / 'scaling.png', dpi=160) plt.close(fig) fig, axes = plt.subplots(2, 2, figsize=(12, 8)) panels = [ ('api-cpu', ['two-small', 'two'], 'all_request_p95_ms', ['API 1 CPU', 'API 2 CPU'], '2 API, 40 RPS: CPU increase', 'All-request p95, ms', 40), ('pg-cpu', ['pg0.5', 'pg1'], 'p95_ms', ['PG 0.5 CPU', 'PG 1 CPU'], '4 API, 60 RPS: database quota', 'Successful-request p95, ms', 60), ('pool-limit', ['pool60', 'pool32'], 'successful_rps', ['60 connections', '32 connections'], '4 API, 100 scheduled RPS: pool limit', 'Successful requests/s', 100), ('pg-upper-cpu', ['pg1', 'pg2'], 'all_request_p95_ms', ['PG 1 CPU', 'PG 2 CPU'], '2 API, 80 RPS: further PG CPU increase', 'All-request p95, ms', 80), ] for ax, (dataset, variants, field, labels, title, ylabel, rate) in zip(axes.flat, panels): for i, variant in enumerate(variants): group = [r for r in rows if r['dataset'] == dataset and r['variant'] == variant and r['offered_rps'] == rate] assert len(group) == 3 color = '#9333ea' if i == 0 else '#059669' for j, r in enumerate(group): # Older Node series use a different scheduler: show their observations without a retroactive k6 classification. underfed = dataset in {'api-cpu', 'pg-upper-cpu'} and not r['delivery_valid'] ax.scatter(i + (j - 1) * .08, r[field], color=color, marker='x' if underfed else 'o', s=55) value = (sum(r['successful'] for r in group) / sum(r['duration_seconds'] for r in group) if dataset == 'pool-limit' else statistics.median(r[field] for r in group)) ax.hlines(value, i - .17, i + .17, color='#0f172a', linewidth=2) ax.annotate(f'{value:.0f}' if value > 100 else f'{value:.2f}', (i, value), xytext=(12, 8), textcoords='offset points', fontsize=9) ax.set(title=title, ylabel=ylabel, xticks=[0, 1], xticklabels=labels, xlim=(-.4, 1.4)) ax.grid(axis='y', alpha=.25) if field == 'all_request_p95_ms': ax.axhline(1000, color='#64748b', linestyle='--', linewidth=1) maximum = max(r[field] for r in rows if r['dataset'] == dataset and r['variant'] in variants and r['offered_rps'] == rate) ax.set_ylim(0, maximum * 1.2) fig.suptitle('Separate experiments: compare variants within each panel', fontsize=13) fig.text(.5, .012, 'Dots: individual runs; black bars: medians (pool panel: aggregate successful RPS). Crosses: k6 underdelivery. See REPORT.md for scheduler differences.', ha='center', fontsize=8) fig.tight_layout(rect=(0, .04, 1, .95)) fig.savefig(output / 'improvements.png', dpi=160) plt.close(fig) def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('--collect', action='store_true', help='Import the selected raw experiment directories before cleanup') parser.add_argument('--source-root', type=Path, default=ROOT / '.artifacts/scaling') parser.add_argument('--output', type=Path, default=ROOT / 'evidence') parser.add_argument('--input', type=Path, default=ROOT / 'evidence/measurements.json') parser.add_argument('--plot', action='store_true') args = parser.parse_args() args.output.mkdir(parents=True, exist_ok=True) bundle = collect(args.source_root, args.output) if args.collect else json.loads(args.input.read_text(encoding='utf-8')) rows = flatten(bundle) verify(bundle, rows) with (args.output / 'scaling.csv').open('w', encoding='utf-8', newline='') as stream: writer = csv.DictWriter(stream, fieldnames=rows[0].keys()) writer.writeheader() writer.writerows(rows) if args.plot: plot(rows, args.output) print(f'Verified {len(rows)} runs from {len(bundle["manifests"])} source series; compact evidence in {args.output}') if __name__ == '__main__': main()