Files
mtusi2026lab3-fork/scripts/report-evidence.py
qrtx c1fec73ccc perf(bench): add scaling experiments and saturation diagnostics
Compare API and PostgreSQL CPU budgets and connection pool limits. Collect repeatable measurements and generate report aggregates and graphs.
2026-10-29 11:49:51 +03:00

221 lines
14 KiB
Python

#!/usr/bin/env python3
"""Keep report evidence in one JSON, one CSV, two figures and checks.json."""
import argparse
from collections import Counter
import csv
from datetime import datetime, timezone
import hashlib
import json
from pathlib import Path
import statistics
import subprocess
ROOT = Path(__file__).resolve().parents[1]
SERIES = [
('api-cpu', None, 'api-small-growth-20261029'),
('pg-cpu', 'pg0.5', 'diagnostics-v3-linux-20261028'),
('pg-cpu', 'pg1', 'diagnostics-pg1cpu-linux-20261028'),
('pool-limit', 'pool60', 'pool15-four-r100-short-20261028'),
('pool-limit', 'pool32', 'pool8-four-r100-short-20261028'),
('pg-upper-cpu', 'pg1', 'api-small-db-control-20261029/pg1'),
('pg-upper-cpu', 'pg2', 'api-small-db-control-20261029/pg2'),
]
def digest(path):
return hashlib.sha256(path.read_bytes()).hexdigest()
def flatten(bundle):
rows = []
for run in bundle['runs']:
profile, measurement = run['profile'], run['measurement']
all_requests = measurement['all']
diagnostics = run['diagnostics']
resources, pools = diagnostics['resources'], diagnostics['pools']
manifest = bundle['manifests'][run['manifest']]
containers = manifest['containers'][run['configuration']]
apis = [c for c in containers if '-api-' in c['name']]
pg = next(c for c in containers if '-postgres-' in c['name'])
seconds, rate = profile['duration_seconds'], profile['offered_rps']
acquired = sum(p['acquisitions'] for p in pools.values())
fraction = all_requests['attempted'] / (seconds * rate)
latency = all_requests['successful_latency_ms']
all_latency = run.get('all_latency_ms') or {}
rows.append({
'dataset': run['dataset'], 'variant': run['variant'], 'label': run['label'],
'replicas': len(apis), 'api_cpu': sum(c['nano_cpus'] for c in apis) / 1e9,
'api_ram_mib': sum(c['memory_bytes'] for c in apis) / 1024**2,
'pool_total': sum(int(c['pool_max']) for c in apis),
'pg_cpu': pg['nano_cpus'] / 1e9, 'pg_ram_mib': pg['memory_bytes'] / 1024**2,
'offered_rps': rate, 'duration_seconds': seconds,
'warmup_seconds': profile.get('warmup_seconds', manifest['warmup_seconds']),
'attempted': all_requests['attempted'], 'successful': all_requests['successful'],
'errors': all_requests['errors'], 'successful_rps': all_requests['successful'] / seconds,
'attempted_rps': all_requests['attempted'] / seconds,
'delivery_fraction': fraction, 'delivery_valid': .98 <= fraction <= 1.02,
'p50_ms': latency['p50'], 'p95_ms': latency['p95'], 'p99_ms': latency['p99'],
'all_request_p95_ms': all_latency.get('p95'),
'api_cpu_cores': sum(r.get('mean_cpu_cores', 0) for name, r in resources.items() if name.startswith('api-')),
'api_peak_memory_mib_sum': sum(r.get('peak_memory_mib', 0) for name, r in resources.items() if name.startswith('api-')),
'api_max_throttle_percent': max((p.get('throttled_period_fraction', 0) for name, p in resources.items() if name.startswith('api-')), default=0) * 100,
'pg_cpu_cores': resources['postgres'].get('mean_cpu_cores'),
'pg_peak_memory_mib': resources['postgres'].get('peak_memory_mib'),
'pg_throttle_percent': resources['postgres'].get('throttled_period_fraction', 0) * 100,
'host_cpu_pressure_percent': diagnostics['host'].get('cpu_some_fraction', 0) * 100,
'acquisition_mean_ms': sum(p['mean_acquisition_ms'] * p['acquisitions'] for p in pools.values()) / max(acquired, 1),
'acquisition_errors': sum(p['acquisition_errors'] for p in pools.values()),
'pool_max_waiting': max((p['max_waiting'] for p in pools.values()), default=0),
'nginx_min_ready': diagnostics['nginx']['min_ready'],
'diagnostics_complete': diagnostics['diagnostics_complete'],
})
return rows
def collect(source, output):
manifests, runs = {}, []
for dataset, variant, directory in SERIES:
path = source / directory
manifest = json.loads((path / 'manifest.json').read_text(encoding='utf-8'))
manifests[directory] = manifest
for label in manifest['runs']:
location = path / label
benchmark = json.loads((location / 'benchmark.json').read_text(encoding='utf-8'))
summary = json.loads((location / 'summary.json').read_text(encoding='utf-8'))
configuration = next(p for p in manifest['profiles'] if label.startswith(p + '-r'))
runs.append({
'dataset': dataset, 'variant': variant or configuration, 'configuration': configuration,
'manifest': directory, 'label': label,
'profile': benchmark['profile'], 'measurement': benchmark['measurement'],
'timing': benchmark.get('timing'), 'client': benchmark.get('client'),
'all_latency_ms': benchmark.get('all_latency_ms'),
'criterion': summary.get('criterion'),
'diagnostics': {key: summary[key] for key in
('resources', 'pools', 'host', 'postgres', 'nginx', 'limits_reached', 'diagnostics_complete')},
'source_hashes': {'benchmark': digest(location / 'benchmark.json'),
'summary': digest(location / 'summary.json')},
})
sources = sorted([p for p in (ROOT / 'src').rglob('*.ts') if 'generated' not in p.parts]
+ list((ROOT / 'prisma').rglob('*.sql'))
+ list((ROOT / 'configs').glob('*.yaml'))
+ [ROOT / p for p in ('compose.yaml', 'Dockerfile', 'infra/nginx.conf', 'infra/start-nginx.sh', 'pnpm-lock.yaml')])
host_path = source / 'diagnostics-pg1cpu-full-20261028/host-environment.json'
host = json.loads(host_path.read_text(encoding='utf-8'))
bundle = {
'schema_version': 1, 'recorded_at': datetime.now(timezone.utc).isoformat(),
'git_base': subprocess.check_output(['git', 'rev-parse', 'HEAD'], cwd=ROOT, text=True).strip(),
'revision_note': 'Implementation is in the working tree; git_base identifies the scaffold, not a committed implementation.',
'retained_data': 'Per-run HTTP aggregates, profiles, CPU/RAM/pool/SQL-state diagnostics, original manifests and source hashes. Request samples, stdout/stderr, traces and duplicate documentation omitted.',
'hardware': {'processor': host['processor'], 'windows_total_visible_kib': host['windows_memory_kib']['total_visible']},
'application_source_sha256': {p.relative_to(ROOT).as_posix(): digest(p) for p in sources},
'manifests': manifests, 'runs': runs,
}
(output / 'measurements.json').write_text(json.dumps(bundle, ensure_ascii=False, indent=2) + '\n', encoding='utf-8')
return bundle
def verify(bundle, rows):
counts = Counter(r['dataset'] for r in rows)
assert counts == {'api-cpu': 36, 'pg-cpu': 6, 'pool-limit': 6, 'pg-upper-cpu': 6}, counts
assert len({(r['dataset'], r['variant'], r['label']) for r in rows}) == 54
assert all(r['attempted'] == r['successful'] + r['errors'] for r in rows)
assert all(r['diagnostics_complete'] for r in rows)
matrix = [r for r in rows if r['dataset'] == 'api-cpu']
assert all(r['pool_total'] == 32 and r['api_ram_mib'] == 1024 and r['pg_cpu'] == 1 for r in matrix)
assert all(n == 3 for n in Counter((r['variant'], r['offered_rps']) for r in matrix).values())
for name, expected in [('pg-cpu', {'pg0.5': .5, 'pg1': 1}),
('pg-upper-cpu', {'pg1': 1, 'pg2': 2})]:
assert all(r['pg_cpu'] == expected[r['variant']] for r in rows if r['dataset'] == name)
assert {r['pool_total'] for r in rows if r['dataset'] == 'pool-limit'} == {32, 60}
assert len(bundle['manifests']) == 7
def plot(rows, output):
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
matrix = [r for r in rows if r['dataset'] == 'api-cpu']
fig, axes = plt.subplots(2, 2, figsize=(12, 8))
for cpu, color in ((1, '#9333ea'), (2, '#059669')):
for ax, rate, field, title, ylabel in (
(axes[0, 0], 40, 'all_request_p95_ms', '40 scheduled RPS: all-request latency', 'Median p95, ms'),
(axes[0, 1], 80, 'successful_rps', '80 scheduled RPS: successful throughput', 'Median successful requests/s'),
(axes[1, 0], 40, 'api_max_throttle_percent', '40 RPS: most throttled API', 'Median throttled periods, %'),
(axes[1, 1], 80, 'delivery_fraction', '80 RPS: actual load delivery', 'Median started / scheduled, %'),
):
groups = [[r for r in matrix if r['api_cpu'] == cpu and r['replicas'] == n and r['offered_rps'] == rate] for n in (1, 2, 4)]
scale = 100 if field == 'delivery_fraction' else 1
values = [statistics.median(r[field] for r in g) * scale for g in groups]
ax.plot([1, 2, 4], values, marker='o', color=color, label=f'API total {cpu} CPU / 1 GiB')
ax.fill_between([1, 2, 4], [min(r[field] for r in g) * scale for g in groups],
[max(r[field] for r in g) * scale for g in groups], color=color, alpha=.1)
ax.set(title=title, xlabel='API replicas', ylabel=ylabel, xticks=[1, 2, 4])
ax.grid(alpha=.25)
ax.legend(fontsize=8)
axes[0, 0].axhline(1000, color='#64748b', linestyle='--')
axes[1, 1].axhline(98, color='#64748b', linestyle='--')
fig.suptitle('Fixed budgets and CPU growth; PG 1 CPU / 512 MiB; total pool 32', fontsize=13)
fig.text(.5, .012, 'All repeats shown, including underfed runs. Lines: medians; bands: min/max, not confidence intervals. No measured plateau claim.', ha='center', fontsize=8)
fig.tight_layout(rect=(0, .04, 1, .95))
fig.savefig(output / 'scaling.png', dpi=160)
plt.close(fig)
fig, axes = plt.subplots(2, 2, figsize=(12, 8))
panels = [
('api-cpu', ['two-small', 'two'], 'all_request_p95_ms', ['API 1 CPU', 'API 2 CPU'], '2 API, 40 RPS: CPU increase', 'All-request p95, ms', 40),
('pg-cpu', ['pg0.5', 'pg1'], 'p95_ms', ['PG 0.5 CPU', 'PG 1 CPU'], '4 API, 60 RPS: database quota', 'Successful-request p95, ms', 60),
('pool-limit', ['pool60', 'pool32'], 'successful_rps', ['60 connections', '32 connections'], '4 API, 100 scheduled RPS: pool limit', 'Successful requests/s', 100),
('pg-upper-cpu', ['pg1', 'pg2'], 'all_request_p95_ms', ['PG 1 CPU', 'PG 2 CPU'], '2 API, 80 RPS: further PG CPU increase', 'All-request p95, ms', 80),
]
for ax, (dataset, variants, field, labels, title, ylabel, rate) in zip(axes.flat, panels):
for i, variant in enumerate(variants):
group = [r for r in rows if r['dataset'] == dataset and r['variant'] == variant and r['offered_rps'] == rate]
assert len(group) == 3
color = '#9333ea' if i == 0 else '#059669'
for j, r in enumerate(group):
# Older Node series use a different scheduler: show their observations without a retroactive k6 classification.
underfed = dataset in {'api-cpu', 'pg-upper-cpu'} and not r['delivery_valid']
ax.scatter(i + (j - 1) * .08, r[field], color=color, marker='x' if underfed else 'o', s=55)
value = (sum(r['successful'] for r in group) / sum(r['duration_seconds'] for r in group)
if dataset == 'pool-limit' else statistics.median(r[field] for r in group))
ax.hlines(value, i - .17, i + .17, color='#0f172a', linewidth=2)
ax.annotate(f'{value:.0f}' if value > 100 else f'{value:.2f}', (i, value),
xytext=(12, 8), textcoords='offset points', fontsize=9)
ax.set(title=title, ylabel=ylabel, xticks=[0, 1], xticklabels=labels, xlim=(-.4, 1.4))
ax.grid(axis='y', alpha=.25)
if field == 'all_request_p95_ms':
ax.axhline(1000, color='#64748b', linestyle='--', linewidth=1)
maximum = max(r[field] for r in rows if r['dataset'] == dataset and r['variant'] in variants and r['offered_rps'] == rate)
ax.set_ylim(0, maximum * 1.2)
fig.suptitle('Separate experiments: compare variants within each panel', fontsize=13)
fig.text(.5, .012, 'Dots: individual runs; black bars: medians (pool panel: aggregate successful RPS). Crosses: k6 underdelivery. See REPORT.md for scheduler differences.', ha='center', fontsize=8)
fig.tight_layout(rect=(0, .04, 1, .95))
fig.savefig(output / 'improvements.png', dpi=160)
plt.close(fig)
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--collect', action='store_true', help='Import the selected raw experiment directories before cleanup')
parser.add_argument('--source-root', type=Path, default=ROOT / '.artifacts/scaling')
parser.add_argument('--output', type=Path, default=ROOT / 'evidence')
parser.add_argument('--input', type=Path, default=ROOT / 'evidence/measurements.json')
parser.add_argument('--plot', action='store_true')
args = parser.parse_args()
args.output.mkdir(parents=True, exist_ok=True)
bundle = collect(args.source_root, args.output) if args.collect else json.loads(args.input.read_text(encoding='utf-8'))
rows = flatten(bundle)
verify(bundle, rows)
with (args.output / 'scaling.csv').open('w', encoding='utf-8', newline='') as stream:
writer = csv.DictWriter(stream, fieldnames=rows[0].keys())
writer.writeheader()
writer.writerows(rows)
if args.plot:
plot(rows, args.output)
print(f'Verified {len(rows)} runs from {len(bundle["manifests"])} source series; compact evidence in {args.output}')
if __name__ == '__main__':
main()