Files
mtusi2026lab3-fork/scripts/run-saturation.py
qrtx c1fec73ccc perf(bench): add scaling experiments and saturation diagnostics
Compare API and PostgreSQL CPU budgets and connection pool limits. Collect repeatable measurements and generate report aggregates and graphs.
2026-10-29 11:49:51 +03:00

292 lines
18 KiB
Python

#!/usr/bin/env python3
"""Bracket an empirical SLO boundary with k6; preserve separate probe/confirmation data."""
import argparse
from datetime import datetime, timezone
import hashlib
import importlib.util
import json
import os
from pathlib import Path
import subprocess
import time
import uuid
ROOT = Path(__file__).resolve().parents[1]
PROFILE_REPLICAS = {'one': 1, 'two': 2, 'four': 4, 'four-large': 4,
'one-small': 1, 'two-small': 2, 'four-small': 4}
spec = importlib.util.spec_from_file_location('diagnostics', ROOT/'scripts/run-scaling-diagnostics.py')
diag = importlib.util.module_from_spec(spec)
spec.loader.exec_module(diag)
def call(args, timeout=180, env=None):
result = subprocess.run(args, cwd=ROOT, capture_output=True, text=True,
encoding='utf-8', errors='replace', timeout=timeout, env=env)
if result.returncode:
raise RuntimeError(f'{args[0]} {args[1]}: {result.stderr[-1800:]}')
return result.stdout.strip()
def watch(container, role, directory):
stop = '/tmp/saturation-' + uuid.uuid4().hex + '.stop'
output = (directory/f'{role}.jsonl').open('wb')
error = (directory/f'{role}.stderr').open('wb')
process = subprocess.Popen(['docker', 'exec', '-i', container, 'sh', '-s', role, stop],
cwd=ROOT, stdin=subprocess.PIPE, stdout=output, stderr=error)
process.stdin.write((ROOT/'scripts/diagnostics/observer.sh').read_bytes())
process.stdin.close()
return container, role, stop, process, output, error
def stop_watch(worker, directory):
container, role, stop, process, output, error = worker
try:
call(['docker', 'exec', container, 'touch', stop], timeout=15)
process.wait(timeout=15)
finally:
if process.poll() is None:
process.terminate()
process.wait(timeout=10)
output.close()
error.close()
if role == 'postgres':
call(['docker', 'cp', f'{container}:{stop}.pg.jsonl', str(directory/'postgres-sql.jsonl')])
call(['docker', 'exec', container, 'rm', '-f', stop+'.pg.jsonl'])
def classify(benchmark):
m = benchmark['measurement']
count = m['all']['attempted']
ratio = count/max(m['scheduled_slots'], 1)
error_fraction = m['all']['errors']/max(count, 1)
p95 = benchmark['all_latency_ms']['p95']
delivered = 0.98 <= ratio <= 1.02 and m['client_dropped_slots']/max(m['scheduled_slots'], 1) <= 0.02
slo = count > 0 and error_fraction <= 0.01 and p95 is not None and p95 <= 1000
return {'delivery_fraction': ratio, 'error_fraction': error_fraction,
'all_request_p95_ms': p95, 'delivery_valid': delivered, 'slo_pass': slo,
'classification': 'pass' if delivered and slo else 'slo_fail' if delivered else 'generator_limited'}
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--output', required=True, type=Path)
parser.add_argument('--image', default='grafana/k6:1.3.0')
parser.add_argument('--profiles', nargs='+', choices=list(PROFILE_REPLICAS), default=['one', 'two', 'four'])
parser.add_argument('--probe-seconds', type=int, default=15)
parser.add_argument('--confirm-seconds', type=int, default=30)
parser.add_argument('--warmup-seconds', type=int, default=10,
help='Mixed-load warmup before each confirmation run')
parser.add_argument('--calibration-only', action='store_true')
parser.add_argument('--confirm-rates', nargs='+', type=int, help='Confirm these rates directly, skipping coarse search')
parser.add_argument('--no-expand', action='store_true',
help='With --confirm-rates, run only the requested rates for a controlled comparison')
args = parser.parse_args()
if args.no_expand and not args.confirm_rates:
parser.error('--no-expand requires --confirm-rates')
if args.warmup_seconds < 1:
parser.error('--warmup-seconds must be positive')
out = args.output.resolve()
out.mkdir(parents=True, exist_ok=False)
manifest = {'started': datetime.now(timezone.utc).isoformat(), 'profiles': args.profiles,
'client': 'container', 'engine': 'k6', 'duration_seconds': args.confirm_seconds,
'warmup_seconds': args.warmup_seconds, 'probe_duration_seconds': args.probe_seconds, 'probe_warmup_seconds': 5,
'repetitions': 3, 'rates': [], 'containers': {}, 'runs': [], 'boundaries': {},
'stopped_profiles': {}, 'observer_version': 3,
'criterion': 'delivery 98..102%, dropped iterations <=2%, errors <=1%, all-request p95 <=1000 ms',
'pool_budget': 32, 'notes': ['Local empirical SLO boundary, not universal maximum throughput.',
'Native sequential warmup/measurement scenarios at target-rate 80/20; custom counters belong only to measurement.',
'Generator shares Docker VM CPU; calibration does not eliminate competition during API load.',
'Probe durations differ; use confirmation rows for reported bounds.'],
'source_sha256': {p: hashlib.sha256((ROOT/p).read_bytes()).hexdigest() for p in
['scripts/saturation.k6.js', 'scripts/saturation-mock.cjs', 'scripts/run-saturation.py',
'scripts/run-scaling-diagnostics.py', 'scripts/diagnostics/pool.cjs', 'scripts/diagnostics/observer.sh',
'compose.yaml', 'test/compose.diagnostics.yaml',
*[f'configs/{profile}.yaml' for profile in args.profiles]]}}
save = lambda: (out/'manifest.json').write_text(json.dumps(manifest, indent=2)+'\n', encoding='utf-8')
manifest['requested_confirmation_rates'] = args.confirm_rates
manifest['expand_confirmation_rates'] = not args.no_expand
for name in manifest['source_sha256']:
target = out/'sources'/name
target.parent.mkdir(parents=True, exist_ok=True)
target.write_bytes((ROOT/name).read_bytes())
save()
nginx = json.loads(call(['docker', 'inspect', call(diag.compose('two', False)+['ps', '-q', 'nginx'])]))[0]
network = next(iter(nginx['NetworkSettings']['Networks']))
api_id = call(diag.compose('two', False)+['ps', '-q', 'api']).split()[0]
api_image = json.loads(call(['docker', 'inspect', api_id]))[0]['Image']
suffix = uuid.uuid4().hex
generator = 'lab3-saturation-client-' + suffix
mock = 'lab3-saturation-mock-' + suffix
generator_watch = None
counter = 0
manifest['generator_container'] = generator
manifest['mock_container'] = mock
save()
def run(profile, rate, phase, seconds, warm, expected, base='http://nginx'):
nonlocal counter
if (out/'CANCEL').exists():
raise InterruptedError('Cancellation requested; restoring ordinary profile')
counter += 1
label = f'{profile}-r{rate}-{phase}-{counter}'
directory = out/label
directory.mkdir()
print(f'START {label} ({seconds}s)', flush=True)
envs = {'BASE_URL': base, 'RATE': str(rate), 'SECONDS': str(seconds), 'WARMUP': str(warm),
'EXPECTED': str(expected), 'LABEL': label, 'PHASE': phase,
'OUTPUT': f'/results/{label}/benchmark.json', 'CALIBRATION': str(int(profile == 'calibration'))}
command = ['docker', 'exec']
for k, v in envs.items():
command += ['-e', f'{k}={v}']
command += [generator, 'k6', 'run', '--quiet', '/scripts/saturation.k6.js']
started = datetime.now(timezone.utc).isoformat()
result = subprocess.run(command, cwd=ROOT, capture_output=True, text=True,
encoding='utf-8', errors='replace', timeout=seconds+warm+150)
(directory/'execution.json').write_text(json.dumps({'returncode': result.returncode,
'started': started, 'ended': datetime.now(timezone.utc).isoformat(), 'command': command}, indent=2)+'\n')
(directory/'k6.stdout').write_text(result.stdout, encoding='utf-8')
(directory/'k6.stderr').write_text(result.stderr, encoding='utf-8')
if result.returncode or not (directory/'benchmark.json').exists():
raise RuntimeError(f'{label}: k6 execution failed: {result.stderr[-1000:]}')
b = json.loads((directory/'benchmark.json').read_text(encoding='utf-8'))
if not b['timing']['measurement_started_epoch_ms']:
raise RuntimeError(f'{label}: no measured requests')
verdict = classify(b)
(directory/'criterion.json').write_text(json.dumps(verdict, indent=2)+'\n')
print(f"DONE {label}: {verdict['classification']} delivered={verdict['delivery_fraction']:.3f} "
f"RPS={b['measurement']['all']['successful_rps']:.2f} p95={verdict['all_request_p95_ms']:.0f}ms "
f"errors={b['measurement']['all']['errors']}", flush=True)
if profile != 'calibration':
manifest['runs'].append(label)
manifest['rates'] = sorted(set(manifest['rates']+[rate]))
save()
time.sleep(1 if verdict['classification'] == 'pass' else 5)
return label, verdict
try:
call(['docker', 'run', '-d', '--name', generator, '--network', network,
'-v', f'{ROOT / "scripts"}:/scripts:ro', '-v', f'{out}:/results',
'--entrypoint', 'sh', args.image, '-c', 'while :; do sleep 3600; done'])
manifest['generator_image'] = json.loads(call(['docker', 'inspect', generator]))[0]['Image']
manifest['generator_version'] = call(['docker', 'exec', generator, 'k6', 'version'])
manifest['docker'] = json.loads(call(['docker', 'info', '--format',
'{"cpus":{{.NCPU}},"memory_bytes":{{.MemTotal}},"version":{{json .ServerVersion}}}']))
generator_watch = watch(generator, 'generator', out)
call(['docker', 'run', '-d', '--name', mock, '--network', network,
'-v', f'{ROOT / "scripts/saturation-mock.cjs"}:/mock.cjs:ro',
'--entrypoint', 'node', api_image, '/mock.cjs'])
calibrations = []
for calibration_rate in [400, 200, 100]:
label, calibration = run('calibration', calibration_rate, 'calibration', 10, 3, 1, f'http://{mock}:8080')
calibrations.append({'label': label, 'rate': calibration_rate, **calibration})
if calibration['classification'] == 'pass': break
manifest['calibration_attempts'] = calibrations
manifest['calibration'] = {'label': label, **calibration}
low_label, low_calibration = run('calibration', 40, 'calibration-low-rate', 10, 3, 1, f'http://{mock}:8080')
manifest['calibration_low_rate'] = {'label': low_label, **low_calibration}
save()
call(['docker', 'rm', '-f', mock])
if calibration['classification'] != 'pass' or low_calibration['classification'] != 'pass':
raise RuntimeError('Generator calibration failed; do not claim API saturation')
if args.calibration_only:
return
for profile in args.profiles:
profile_started = datetime.now(timezone.utc).isoformat()
n = PROFILE_REPLICAS[profile]
print(f'SWITCH {profile}, pool={32//n} per replica', flush=True)
env = {**os.environ, 'DB_POOL_MAX': str(32//n)}
call(diag.compose(profile)+['up', '-d', '--no-build', '--no-deps', '--wait', '--wait-timeout', '120', 'api'], env=env)
time.sleep(6)
containers = {role: call(diag.compose(profile)+['ps', '-q', role]).split() for role in ['api', 'postgres', 'nginx', 'redis']}
details = json.loads(call(['docker', 'inspect', *[c for group in containers.values() for c in group]]))
manifest['containers'][profile] = [{'id': c['Id'], 'name': c['Name'], 'image': c['Image'],
'nano_cpus': c['HostConfig']['NanoCpus'], 'memory_bytes': c['HostConfig']['Memory'],
'pool_max': next((e.split('=', 1)[1] for e in c['Config']['Env'] if e.startswith('DB_POOL_MAX=')), None)} for c in details]
profile_dir = out/f'observers-{profile}'
profile_dir.mkdir()
workers = []
first = len(manifest['runs'])
save()
try:
for role in ['postgres', 'nginx', 'redis']:
workers.append(watch(containers[role][0], role, profile_dir))
low = None
high = None
for rate in ([] if args.confirm_rates else [40, 80, 120, 160, 240, 320, 480]):
_, v = run(profile, rate, 'probe', args.probe_seconds, 5, n)
if v['classification'] == 'pass': low = rate
else:
high = rate
break
if low is None and not args.confirm_rates:
for rate in [20, 10]:
_, v = run(profile, rate, 'probe', args.probe_seconds, 5, n)
if v['classification'] == 'pass': low = rate; break
high = rate
if low is not None and high is not None:
for _ in range(2):
if high-low <= 20: break
middle = ((low+high)//20)*10
_, v = run(profile, middle, 'probe', args.probe_seconds, 5, n)
if v['classification'] == 'pass': low = middle
else: high = middle
confirmations = {}
for repeat in range(3):
for rate in sorted(set(args.confirm_rates or [r for r in [low, high] if r is not None])):
_, v = run(profile, rate, 'confirmation', args.confirm_seconds, args.warmup_seconds, n)
confirmations.setdefault(rate, []).append(v)
for _ in range(0 if args.no_expand else 2):
good = [r for r, vv in confirmations.items() if all(v['classification'] == 'pass' for v in vv)]
bad = [r for r, vv in confirmations.items() if sum(v['classification'] == 'slo_fail' for v in vv) >= 2]
if good and bad and max(good) < min(bad): break
if any(v['classification'] == 'generator_limited' for vv in confirmations.values() for v in vv): break
rate = max(10, min(confirmations)-20) if not good else max(confirmations)+20
if rate in confirmations: break
confirmations[rate] = [run(profile, rate, 'confirmation', args.confirm_seconds, args.warmup_seconds, n)[1] for _ in range(3)]
good = [r for r, vv in confirmations.items() if all(v['classification'] == 'pass' for v in vv)]
bad = [r for r, vv in confirmations.items() if sum(v['classification'] == 'slo_fail' for v in vv) >= 2]
manifest['boundaries'][profile] = {'confirmed_good': max(good) if good else None,
'confirmed_bad': min(bad) if bad else None, 'confirmations': confirmations,
'bracketed': bool(good and bad and max(good) < min(bad))}
print(f"BOUNDARY {profile}: {manifest['boundaries'][profile]['confirmed_good']} .. "
f"{manifest['boundaries'][profile]['confirmed_bad']} RPS", flush=True)
finally:
for worker in workers: stop_watch(worker, profile_dir)
log_result = subprocess.run(['docker', 'logs', '--timestamps', '--since', profile_started,
containers['nginx'][0]], cwd=ROOT, capture_output=True, text=True,
encoding='utf-8', errors='replace', timeout=30)
events = [line for line in (log_result.stdout+'\n'+log_result.stderr).splitlines()
if any(term in line for term in ['worker_connections', 'no live upstreams',
'upstream timed out', 'download timed out', 'signal process started'])]
(profile_dir/'nginx-events.log').write_text('\n'.join(events)+'\n', encoding='utf-8')
for index, container in enumerate(containers['api'], 1):
call(['docker', 'cp', f'{container}:/tmp/scaling-pool.jsonl', str(profile_dir/f'api-{index}.jsonl')])
for label in manifest['runs'][first:]:
directory = out/label
for p in profile_dir.glob('*.jsonl'):
(directory/p.name).write_bytes(p.read_bytes())
(directory/'generator.jsonl').write_bytes((out/'generator.jsonl').read_bytes())
b = json.loads((directory/'benchmark.json').read_text())
summary = diag.summarize(directory, b)
samples = [r for r in diag.rows(directory/'generator.jsonl') if r['kind'] == 'cgroup'
and b['timing']['measurement_started_epoch_ms'] <= r['time_ms'] <= b['timing']['measurement_scheduled_end_epoch_ms']]
summary['resources']['generator'] = diag.resource_summary(samples)
summary['all_latency_ms'] = b['all_latency_ms']
summary['criterion'] = json.loads((directory/'criterion.json').read_text())
summary['diagnostics_complete'] = len(summary['pools']) == n and all(v['samples'] >= 2 for v in summary['resources'].values())
(directory/'summary.json').write_text(json.dumps(summary, indent=2)+'\n')
save()
finally:
if generator_watch: stop_watch(generator_watch, out)
for name in [mock, generator]:
subprocess.run(['docker', 'rm', '-f', name], cwd=ROOT, capture_output=True, timeout=30)
if not args.calibration_only:
print('RESTORE normal two profile, original pool setting', flush=True)
call(diag.compose('two', False)+['up', '-d', '--no-build', '--no-deps', '--wait', '--wait-timeout', '120', 'api'])
manifest['finished'] = datetime.now(timezone.utc).isoformat()
save()
if __name__ == '__main__':
main()