forked from mtusi2026/mtusi2026lab3
Compare API and PostgreSQL CPU budgets and connection pool limits. Collect repeatable measurements and generate report aggregates and graphs.
292 lines
18 KiB
Python
292 lines
18 KiB
Python
#!/usr/bin/env python3
|
|
"""Bracket an empirical SLO boundary with k6; preserve separate probe/confirmation data."""
|
|
import argparse
|
|
from datetime import datetime, timezone
|
|
import hashlib
|
|
import importlib.util
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import subprocess
|
|
import time
|
|
import uuid
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
PROFILE_REPLICAS = {'one': 1, 'two': 2, 'four': 4, 'four-large': 4,
|
|
'one-small': 1, 'two-small': 2, 'four-small': 4}
|
|
spec = importlib.util.spec_from_file_location('diagnostics', ROOT/'scripts/run-scaling-diagnostics.py')
|
|
diag = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(diag)
|
|
|
|
|
|
def call(args, timeout=180, env=None):
|
|
result = subprocess.run(args, cwd=ROOT, capture_output=True, text=True,
|
|
encoding='utf-8', errors='replace', timeout=timeout, env=env)
|
|
if result.returncode:
|
|
raise RuntimeError(f'{args[0]} {args[1]}: {result.stderr[-1800:]}')
|
|
return result.stdout.strip()
|
|
|
|
|
|
def watch(container, role, directory):
|
|
stop = '/tmp/saturation-' + uuid.uuid4().hex + '.stop'
|
|
output = (directory/f'{role}.jsonl').open('wb')
|
|
error = (directory/f'{role}.stderr').open('wb')
|
|
process = subprocess.Popen(['docker', 'exec', '-i', container, 'sh', '-s', role, stop],
|
|
cwd=ROOT, stdin=subprocess.PIPE, stdout=output, stderr=error)
|
|
process.stdin.write((ROOT/'scripts/diagnostics/observer.sh').read_bytes())
|
|
process.stdin.close()
|
|
return container, role, stop, process, output, error
|
|
|
|
|
|
def stop_watch(worker, directory):
|
|
container, role, stop, process, output, error = worker
|
|
try:
|
|
call(['docker', 'exec', container, 'touch', stop], timeout=15)
|
|
process.wait(timeout=15)
|
|
finally:
|
|
if process.poll() is None:
|
|
process.terminate()
|
|
process.wait(timeout=10)
|
|
output.close()
|
|
error.close()
|
|
if role == 'postgres':
|
|
call(['docker', 'cp', f'{container}:{stop}.pg.jsonl', str(directory/'postgres-sql.jsonl')])
|
|
call(['docker', 'exec', container, 'rm', '-f', stop+'.pg.jsonl'])
|
|
|
|
|
|
def classify(benchmark):
|
|
m = benchmark['measurement']
|
|
count = m['all']['attempted']
|
|
ratio = count/max(m['scheduled_slots'], 1)
|
|
error_fraction = m['all']['errors']/max(count, 1)
|
|
p95 = benchmark['all_latency_ms']['p95']
|
|
delivered = 0.98 <= ratio <= 1.02 and m['client_dropped_slots']/max(m['scheduled_slots'], 1) <= 0.02
|
|
slo = count > 0 and error_fraction <= 0.01 and p95 is not None and p95 <= 1000
|
|
return {'delivery_fraction': ratio, 'error_fraction': error_fraction,
|
|
'all_request_p95_ms': p95, 'delivery_valid': delivered, 'slo_pass': slo,
|
|
'classification': 'pass' if delivered and slo else 'slo_fail' if delivered else 'generator_limited'}
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument('--output', required=True, type=Path)
|
|
parser.add_argument('--image', default='grafana/k6:1.3.0')
|
|
parser.add_argument('--profiles', nargs='+', choices=list(PROFILE_REPLICAS), default=['one', 'two', 'four'])
|
|
parser.add_argument('--probe-seconds', type=int, default=15)
|
|
parser.add_argument('--confirm-seconds', type=int, default=30)
|
|
parser.add_argument('--warmup-seconds', type=int, default=10,
|
|
help='Mixed-load warmup before each confirmation run')
|
|
parser.add_argument('--calibration-only', action='store_true')
|
|
parser.add_argument('--confirm-rates', nargs='+', type=int, help='Confirm these rates directly, skipping coarse search')
|
|
parser.add_argument('--no-expand', action='store_true',
|
|
help='With --confirm-rates, run only the requested rates for a controlled comparison')
|
|
args = parser.parse_args()
|
|
if args.no_expand and not args.confirm_rates:
|
|
parser.error('--no-expand requires --confirm-rates')
|
|
if args.warmup_seconds < 1:
|
|
parser.error('--warmup-seconds must be positive')
|
|
out = args.output.resolve()
|
|
out.mkdir(parents=True, exist_ok=False)
|
|
manifest = {'started': datetime.now(timezone.utc).isoformat(), 'profiles': args.profiles,
|
|
'client': 'container', 'engine': 'k6', 'duration_seconds': args.confirm_seconds,
|
|
'warmup_seconds': args.warmup_seconds, 'probe_duration_seconds': args.probe_seconds, 'probe_warmup_seconds': 5,
|
|
'repetitions': 3, 'rates': [], 'containers': {}, 'runs': [], 'boundaries': {},
|
|
'stopped_profiles': {}, 'observer_version': 3,
|
|
'criterion': 'delivery 98..102%, dropped iterations <=2%, errors <=1%, all-request p95 <=1000 ms',
|
|
'pool_budget': 32, 'notes': ['Local empirical SLO boundary, not universal maximum throughput.',
|
|
'Native sequential warmup/measurement scenarios at target-rate 80/20; custom counters belong only to measurement.',
|
|
'Generator shares Docker VM CPU; calibration does not eliminate competition during API load.',
|
|
'Probe durations differ; use confirmation rows for reported bounds.'],
|
|
'source_sha256': {p: hashlib.sha256((ROOT/p).read_bytes()).hexdigest() for p in
|
|
['scripts/saturation.k6.js', 'scripts/saturation-mock.cjs', 'scripts/run-saturation.py',
|
|
'scripts/run-scaling-diagnostics.py', 'scripts/diagnostics/pool.cjs', 'scripts/diagnostics/observer.sh',
|
|
'compose.yaml', 'test/compose.diagnostics.yaml',
|
|
*[f'configs/{profile}.yaml' for profile in args.profiles]]}}
|
|
save = lambda: (out/'manifest.json').write_text(json.dumps(manifest, indent=2)+'\n', encoding='utf-8')
|
|
manifest['requested_confirmation_rates'] = args.confirm_rates
|
|
manifest['expand_confirmation_rates'] = not args.no_expand
|
|
for name in manifest['source_sha256']:
|
|
target = out/'sources'/name
|
|
target.parent.mkdir(parents=True, exist_ok=True)
|
|
target.write_bytes((ROOT/name).read_bytes())
|
|
save()
|
|
nginx = json.loads(call(['docker', 'inspect', call(diag.compose('two', False)+['ps', '-q', 'nginx'])]))[0]
|
|
network = next(iter(nginx['NetworkSettings']['Networks']))
|
|
api_id = call(diag.compose('two', False)+['ps', '-q', 'api']).split()[0]
|
|
api_image = json.loads(call(['docker', 'inspect', api_id]))[0]['Image']
|
|
suffix = uuid.uuid4().hex
|
|
generator = 'lab3-saturation-client-' + suffix
|
|
mock = 'lab3-saturation-mock-' + suffix
|
|
generator_watch = None
|
|
counter = 0
|
|
manifest['generator_container'] = generator
|
|
manifest['mock_container'] = mock
|
|
save()
|
|
|
|
def run(profile, rate, phase, seconds, warm, expected, base='http://nginx'):
|
|
nonlocal counter
|
|
if (out/'CANCEL').exists():
|
|
raise InterruptedError('Cancellation requested; restoring ordinary profile')
|
|
counter += 1
|
|
label = f'{profile}-r{rate}-{phase}-{counter}'
|
|
directory = out/label
|
|
directory.mkdir()
|
|
print(f'START {label} ({seconds}s)', flush=True)
|
|
envs = {'BASE_URL': base, 'RATE': str(rate), 'SECONDS': str(seconds), 'WARMUP': str(warm),
|
|
'EXPECTED': str(expected), 'LABEL': label, 'PHASE': phase,
|
|
'OUTPUT': f'/results/{label}/benchmark.json', 'CALIBRATION': str(int(profile == 'calibration'))}
|
|
command = ['docker', 'exec']
|
|
for k, v in envs.items():
|
|
command += ['-e', f'{k}={v}']
|
|
command += [generator, 'k6', 'run', '--quiet', '/scripts/saturation.k6.js']
|
|
started = datetime.now(timezone.utc).isoformat()
|
|
result = subprocess.run(command, cwd=ROOT, capture_output=True, text=True,
|
|
encoding='utf-8', errors='replace', timeout=seconds+warm+150)
|
|
(directory/'execution.json').write_text(json.dumps({'returncode': result.returncode,
|
|
'started': started, 'ended': datetime.now(timezone.utc).isoformat(), 'command': command}, indent=2)+'\n')
|
|
(directory/'k6.stdout').write_text(result.stdout, encoding='utf-8')
|
|
(directory/'k6.stderr').write_text(result.stderr, encoding='utf-8')
|
|
if result.returncode or not (directory/'benchmark.json').exists():
|
|
raise RuntimeError(f'{label}: k6 execution failed: {result.stderr[-1000:]}')
|
|
b = json.loads((directory/'benchmark.json').read_text(encoding='utf-8'))
|
|
if not b['timing']['measurement_started_epoch_ms']:
|
|
raise RuntimeError(f'{label}: no measured requests')
|
|
verdict = classify(b)
|
|
(directory/'criterion.json').write_text(json.dumps(verdict, indent=2)+'\n')
|
|
print(f"DONE {label}: {verdict['classification']} delivered={verdict['delivery_fraction']:.3f} "
|
|
f"RPS={b['measurement']['all']['successful_rps']:.2f} p95={verdict['all_request_p95_ms']:.0f}ms "
|
|
f"errors={b['measurement']['all']['errors']}", flush=True)
|
|
if profile != 'calibration':
|
|
manifest['runs'].append(label)
|
|
manifest['rates'] = sorted(set(manifest['rates']+[rate]))
|
|
save()
|
|
time.sleep(1 if verdict['classification'] == 'pass' else 5)
|
|
return label, verdict
|
|
|
|
try:
|
|
call(['docker', 'run', '-d', '--name', generator, '--network', network,
|
|
'-v', f'{ROOT / "scripts"}:/scripts:ro', '-v', f'{out}:/results',
|
|
'--entrypoint', 'sh', args.image, '-c', 'while :; do sleep 3600; done'])
|
|
manifest['generator_image'] = json.loads(call(['docker', 'inspect', generator]))[0]['Image']
|
|
manifest['generator_version'] = call(['docker', 'exec', generator, 'k6', 'version'])
|
|
manifest['docker'] = json.loads(call(['docker', 'info', '--format',
|
|
'{"cpus":{{.NCPU}},"memory_bytes":{{.MemTotal}},"version":{{json .ServerVersion}}}']))
|
|
generator_watch = watch(generator, 'generator', out)
|
|
call(['docker', 'run', '-d', '--name', mock, '--network', network,
|
|
'-v', f'{ROOT / "scripts/saturation-mock.cjs"}:/mock.cjs:ro',
|
|
'--entrypoint', 'node', api_image, '/mock.cjs'])
|
|
calibrations = []
|
|
for calibration_rate in [400, 200, 100]:
|
|
label, calibration = run('calibration', calibration_rate, 'calibration', 10, 3, 1, f'http://{mock}:8080')
|
|
calibrations.append({'label': label, 'rate': calibration_rate, **calibration})
|
|
if calibration['classification'] == 'pass': break
|
|
manifest['calibration_attempts'] = calibrations
|
|
manifest['calibration'] = {'label': label, **calibration}
|
|
low_label, low_calibration = run('calibration', 40, 'calibration-low-rate', 10, 3, 1, f'http://{mock}:8080')
|
|
manifest['calibration_low_rate'] = {'label': low_label, **low_calibration}
|
|
save()
|
|
call(['docker', 'rm', '-f', mock])
|
|
if calibration['classification'] != 'pass' or low_calibration['classification'] != 'pass':
|
|
raise RuntimeError('Generator calibration failed; do not claim API saturation')
|
|
if args.calibration_only:
|
|
return
|
|
for profile in args.profiles:
|
|
profile_started = datetime.now(timezone.utc).isoformat()
|
|
n = PROFILE_REPLICAS[profile]
|
|
print(f'SWITCH {profile}, pool={32//n} per replica', flush=True)
|
|
env = {**os.environ, 'DB_POOL_MAX': str(32//n)}
|
|
call(diag.compose(profile)+['up', '-d', '--no-build', '--no-deps', '--wait', '--wait-timeout', '120', 'api'], env=env)
|
|
time.sleep(6)
|
|
containers = {role: call(diag.compose(profile)+['ps', '-q', role]).split() for role in ['api', 'postgres', 'nginx', 'redis']}
|
|
details = json.loads(call(['docker', 'inspect', *[c for group in containers.values() for c in group]]))
|
|
manifest['containers'][profile] = [{'id': c['Id'], 'name': c['Name'], 'image': c['Image'],
|
|
'nano_cpus': c['HostConfig']['NanoCpus'], 'memory_bytes': c['HostConfig']['Memory'],
|
|
'pool_max': next((e.split('=', 1)[1] for e in c['Config']['Env'] if e.startswith('DB_POOL_MAX=')), None)} for c in details]
|
|
profile_dir = out/f'observers-{profile}'
|
|
profile_dir.mkdir()
|
|
workers = []
|
|
first = len(manifest['runs'])
|
|
save()
|
|
try:
|
|
for role in ['postgres', 'nginx', 'redis']:
|
|
workers.append(watch(containers[role][0], role, profile_dir))
|
|
low = None
|
|
high = None
|
|
for rate in ([] if args.confirm_rates else [40, 80, 120, 160, 240, 320, 480]):
|
|
_, v = run(profile, rate, 'probe', args.probe_seconds, 5, n)
|
|
if v['classification'] == 'pass': low = rate
|
|
else:
|
|
high = rate
|
|
break
|
|
if low is None and not args.confirm_rates:
|
|
for rate in [20, 10]:
|
|
_, v = run(profile, rate, 'probe', args.probe_seconds, 5, n)
|
|
if v['classification'] == 'pass': low = rate; break
|
|
high = rate
|
|
if low is not None and high is not None:
|
|
for _ in range(2):
|
|
if high-low <= 20: break
|
|
middle = ((low+high)//20)*10
|
|
_, v = run(profile, middle, 'probe', args.probe_seconds, 5, n)
|
|
if v['classification'] == 'pass': low = middle
|
|
else: high = middle
|
|
confirmations = {}
|
|
for repeat in range(3):
|
|
for rate in sorted(set(args.confirm_rates or [r for r in [low, high] if r is not None])):
|
|
_, v = run(profile, rate, 'confirmation', args.confirm_seconds, args.warmup_seconds, n)
|
|
confirmations.setdefault(rate, []).append(v)
|
|
for _ in range(0 if args.no_expand else 2):
|
|
good = [r for r, vv in confirmations.items() if all(v['classification'] == 'pass' for v in vv)]
|
|
bad = [r for r, vv in confirmations.items() if sum(v['classification'] == 'slo_fail' for v in vv) >= 2]
|
|
if good and bad and max(good) < min(bad): break
|
|
if any(v['classification'] == 'generator_limited' for vv in confirmations.values() for v in vv): break
|
|
rate = max(10, min(confirmations)-20) if not good else max(confirmations)+20
|
|
if rate in confirmations: break
|
|
confirmations[rate] = [run(profile, rate, 'confirmation', args.confirm_seconds, args.warmup_seconds, n)[1] for _ in range(3)]
|
|
good = [r for r, vv in confirmations.items() if all(v['classification'] == 'pass' for v in vv)]
|
|
bad = [r for r, vv in confirmations.items() if sum(v['classification'] == 'slo_fail' for v in vv) >= 2]
|
|
manifest['boundaries'][profile] = {'confirmed_good': max(good) if good else None,
|
|
'confirmed_bad': min(bad) if bad else None, 'confirmations': confirmations,
|
|
'bracketed': bool(good and bad and max(good) < min(bad))}
|
|
print(f"BOUNDARY {profile}: {manifest['boundaries'][profile]['confirmed_good']} .. "
|
|
f"{manifest['boundaries'][profile]['confirmed_bad']} RPS", flush=True)
|
|
finally:
|
|
for worker in workers: stop_watch(worker, profile_dir)
|
|
log_result = subprocess.run(['docker', 'logs', '--timestamps', '--since', profile_started,
|
|
containers['nginx'][0]], cwd=ROOT, capture_output=True, text=True,
|
|
encoding='utf-8', errors='replace', timeout=30)
|
|
events = [line for line in (log_result.stdout+'\n'+log_result.stderr).splitlines()
|
|
if any(term in line for term in ['worker_connections', 'no live upstreams',
|
|
'upstream timed out', 'download timed out', 'signal process started'])]
|
|
(profile_dir/'nginx-events.log').write_text('\n'.join(events)+'\n', encoding='utf-8')
|
|
for index, container in enumerate(containers['api'], 1):
|
|
call(['docker', 'cp', f'{container}:/tmp/scaling-pool.jsonl', str(profile_dir/f'api-{index}.jsonl')])
|
|
for label in manifest['runs'][first:]:
|
|
directory = out/label
|
|
for p in profile_dir.glob('*.jsonl'):
|
|
(directory/p.name).write_bytes(p.read_bytes())
|
|
(directory/'generator.jsonl').write_bytes((out/'generator.jsonl').read_bytes())
|
|
b = json.loads((directory/'benchmark.json').read_text())
|
|
summary = diag.summarize(directory, b)
|
|
samples = [r for r in diag.rows(directory/'generator.jsonl') if r['kind'] == 'cgroup'
|
|
and b['timing']['measurement_started_epoch_ms'] <= r['time_ms'] <= b['timing']['measurement_scheduled_end_epoch_ms']]
|
|
summary['resources']['generator'] = diag.resource_summary(samples)
|
|
summary['all_latency_ms'] = b['all_latency_ms']
|
|
summary['criterion'] = json.loads((directory/'criterion.json').read_text())
|
|
summary['diagnostics_complete'] = len(summary['pools']) == n and all(v['samples'] >= 2 for v in summary['resources'].values())
|
|
(directory/'summary.json').write_text(json.dumps(summary, indent=2)+'\n')
|
|
save()
|
|
finally:
|
|
if generator_watch: stop_watch(generator_watch, out)
|
|
for name in [mock, generator]:
|
|
subprocess.run(['docker', 'rm', '-f', name], cwd=ROOT, capture_output=True, timeout=30)
|
|
if not args.calibration_only:
|
|
print('RESTORE normal two profile, original pool setting', flush=True)
|
|
call(diag.compose('two', False)+['up', '-d', '--no-build', '--no-deps', '--wait', '--wait-timeout', '120', 'api'])
|
|
manifest['finished'] = datetime.now(timezone.utc).isoformat()
|
|
save()
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|