#!/usr/bin/env python3 """Bracket an empirical SLO boundary with k6; preserve separate probe/confirmation data.""" import argparse from datetime import datetime, timezone import hashlib import importlib.util import json import os from pathlib import Path import subprocess import time import uuid ROOT = Path(__file__).resolve().parents[1] PROFILE_REPLICAS = {'one': 1, 'two': 2, 'four': 4, 'four-large': 4, 'one-small': 1, 'two-small': 2, 'four-small': 4} spec = importlib.util.spec_from_file_location('diagnostics', ROOT/'scripts/run-scaling-diagnostics.py') diag = importlib.util.module_from_spec(spec) spec.loader.exec_module(diag) def call(args, timeout=180, env=None): result = subprocess.run(args, cwd=ROOT, capture_output=True, text=True, encoding='utf-8', errors='replace', timeout=timeout, env=env) if result.returncode: raise RuntimeError(f'{args[0]} {args[1]}: {result.stderr[-1800:]}') return result.stdout.strip() def watch(container, role, directory): stop = '/tmp/saturation-' + uuid.uuid4().hex + '.stop' output = (directory/f'{role}.jsonl').open('wb') error = (directory/f'{role}.stderr').open('wb') process = subprocess.Popen(['docker', 'exec', '-i', container, 'sh', '-s', role, stop], cwd=ROOT, stdin=subprocess.PIPE, stdout=output, stderr=error) process.stdin.write((ROOT/'scripts/diagnostics/observer.sh').read_bytes()) process.stdin.close() return container, role, stop, process, output, error def stop_watch(worker, directory): container, role, stop, process, output, error = worker try: call(['docker', 'exec', container, 'touch', stop], timeout=15) process.wait(timeout=15) finally: if process.poll() is None: process.terminate() process.wait(timeout=10) output.close() error.close() if role == 'postgres': call(['docker', 'cp', f'{container}:{stop}.pg.jsonl', str(directory/'postgres-sql.jsonl')]) call(['docker', 'exec', container, 'rm', '-f', stop+'.pg.jsonl']) def classify(benchmark): m = benchmark['measurement'] count = m['all']['attempted'] ratio = count/max(m['scheduled_slots'], 1) error_fraction = m['all']['errors']/max(count, 1) p95 = benchmark['all_latency_ms']['p95'] delivered = 0.98 <= ratio <= 1.02 and m['client_dropped_slots']/max(m['scheduled_slots'], 1) <= 0.02 slo = count > 0 and error_fraction <= 0.01 and p95 is not None and p95 <= 1000 return {'delivery_fraction': ratio, 'error_fraction': error_fraction, 'all_request_p95_ms': p95, 'delivery_valid': delivered, 'slo_pass': slo, 'classification': 'pass' if delivered and slo else 'slo_fail' if delivered else 'generator_limited'} def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('--output', required=True, type=Path) parser.add_argument('--image', default='grafana/k6:1.3.0') parser.add_argument('--profiles', nargs='+', choices=list(PROFILE_REPLICAS), default=['one', 'two', 'four']) parser.add_argument('--probe-seconds', type=int, default=15) parser.add_argument('--confirm-seconds', type=int, default=30) parser.add_argument('--warmup-seconds', type=int, default=10, help='Mixed-load warmup before each confirmation run') parser.add_argument('--calibration-only', action='store_true') parser.add_argument('--confirm-rates', nargs='+', type=int, help='Confirm these rates directly, skipping coarse search') parser.add_argument('--no-expand', action='store_true', help='With --confirm-rates, run only the requested rates for a controlled comparison') args = parser.parse_args() if args.no_expand and not args.confirm_rates: parser.error('--no-expand requires --confirm-rates') if args.warmup_seconds < 1: parser.error('--warmup-seconds must be positive') out = args.output.resolve() out.mkdir(parents=True, exist_ok=False) manifest = {'started': datetime.now(timezone.utc).isoformat(), 'profiles': args.profiles, 'client': 'container', 'engine': 'k6', 'duration_seconds': args.confirm_seconds, 'warmup_seconds': args.warmup_seconds, 'probe_duration_seconds': args.probe_seconds, 'probe_warmup_seconds': 5, 'repetitions': 3, 'rates': [], 'containers': {}, 'runs': [], 'boundaries': {}, 'stopped_profiles': {}, 'observer_version': 3, 'criterion': 'delivery 98..102%, dropped iterations <=2%, errors <=1%, all-request p95 <=1000 ms', 'pool_budget': 32, 'notes': ['Local empirical SLO boundary, not universal maximum throughput.', 'Native sequential warmup/measurement scenarios at target-rate 80/20; custom counters belong only to measurement.', 'Generator shares Docker VM CPU; calibration does not eliminate competition during API load.', 'Probe durations differ; use confirmation rows for reported bounds.'], 'source_sha256': {p: hashlib.sha256((ROOT/p).read_bytes()).hexdigest() for p in ['scripts/saturation.k6.js', 'scripts/saturation-mock.cjs', 'scripts/run-saturation.py', 'scripts/run-scaling-diagnostics.py', 'scripts/diagnostics/pool.cjs', 'scripts/diagnostics/observer.sh', 'compose.yaml', 'test/compose.diagnostics.yaml', *[f'configs/{profile}.yaml' for profile in args.profiles]]}} save = lambda: (out/'manifest.json').write_text(json.dumps(manifest, indent=2)+'\n', encoding='utf-8') manifest['requested_confirmation_rates'] = args.confirm_rates manifest['expand_confirmation_rates'] = not args.no_expand for name in manifest['source_sha256']: target = out/'sources'/name target.parent.mkdir(parents=True, exist_ok=True) target.write_bytes((ROOT/name).read_bytes()) save() nginx = json.loads(call(['docker', 'inspect', call(diag.compose('two', False)+['ps', '-q', 'nginx'])]))[0] network = next(iter(nginx['NetworkSettings']['Networks'])) api_id = call(diag.compose('two', False)+['ps', '-q', 'api']).split()[0] api_image = json.loads(call(['docker', 'inspect', api_id]))[0]['Image'] suffix = uuid.uuid4().hex generator = 'lab3-saturation-client-' + suffix mock = 'lab3-saturation-mock-' + suffix generator_watch = None counter = 0 manifest['generator_container'] = generator manifest['mock_container'] = mock save() def run(profile, rate, phase, seconds, warm, expected, base='http://nginx'): nonlocal counter if (out/'CANCEL').exists(): raise InterruptedError('Cancellation requested; restoring ordinary profile') counter += 1 label = f'{profile}-r{rate}-{phase}-{counter}' directory = out/label directory.mkdir() print(f'START {label} ({seconds}s)', flush=True) envs = {'BASE_URL': base, 'RATE': str(rate), 'SECONDS': str(seconds), 'WARMUP': str(warm), 'EXPECTED': str(expected), 'LABEL': label, 'PHASE': phase, 'OUTPUT': f'/results/{label}/benchmark.json', 'CALIBRATION': str(int(profile == 'calibration'))} command = ['docker', 'exec'] for k, v in envs.items(): command += ['-e', f'{k}={v}'] command += [generator, 'k6', 'run', '--quiet', '/scripts/saturation.k6.js'] started = datetime.now(timezone.utc).isoformat() result = subprocess.run(command, cwd=ROOT, capture_output=True, text=True, encoding='utf-8', errors='replace', timeout=seconds+warm+150) (directory/'execution.json').write_text(json.dumps({'returncode': result.returncode, 'started': started, 'ended': datetime.now(timezone.utc).isoformat(), 'command': command}, indent=2)+'\n') (directory/'k6.stdout').write_text(result.stdout, encoding='utf-8') (directory/'k6.stderr').write_text(result.stderr, encoding='utf-8') if result.returncode or not (directory/'benchmark.json').exists(): raise RuntimeError(f'{label}: k6 execution failed: {result.stderr[-1000:]}') b = json.loads((directory/'benchmark.json').read_text(encoding='utf-8')) if not b['timing']['measurement_started_epoch_ms']: raise RuntimeError(f'{label}: no measured requests') verdict = classify(b) (directory/'criterion.json').write_text(json.dumps(verdict, indent=2)+'\n') print(f"DONE {label}: {verdict['classification']} delivered={verdict['delivery_fraction']:.3f} " f"RPS={b['measurement']['all']['successful_rps']:.2f} p95={verdict['all_request_p95_ms']:.0f}ms " f"errors={b['measurement']['all']['errors']}", flush=True) if profile != 'calibration': manifest['runs'].append(label) manifest['rates'] = sorted(set(manifest['rates']+[rate])) save() time.sleep(1 if verdict['classification'] == 'pass' else 5) return label, verdict try: call(['docker', 'run', '-d', '--name', generator, '--network', network, '-v', f'{ROOT / "scripts"}:/scripts:ro', '-v', f'{out}:/results', '--entrypoint', 'sh', args.image, '-c', 'while :; do sleep 3600; done']) manifest['generator_image'] = json.loads(call(['docker', 'inspect', generator]))[0]['Image'] manifest['generator_version'] = call(['docker', 'exec', generator, 'k6', 'version']) manifest['docker'] = json.loads(call(['docker', 'info', '--format', '{"cpus":{{.NCPU}},"memory_bytes":{{.MemTotal}},"version":{{json .ServerVersion}}}'])) generator_watch = watch(generator, 'generator', out) call(['docker', 'run', '-d', '--name', mock, '--network', network, '-v', f'{ROOT / "scripts/saturation-mock.cjs"}:/mock.cjs:ro', '--entrypoint', 'node', api_image, '/mock.cjs']) calibrations = [] for calibration_rate in [400, 200, 100]: label, calibration = run('calibration', calibration_rate, 'calibration', 10, 3, 1, f'http://{mock}:8080') calibrations.append({'label': label, 'rate': calibration_rate, **calibration}) if calibration['classification'] == 'pass': break manifest['calibration_attempts'] = calibrations manifest['calibration'] = {'label': label, **calibration} low_label, low_calibration = run('calibration', 40, 'calibration-low-rate', 10, 3, 1, f'http://{mock}:8080') manifest['calibration_low_rate'] = {'label': low_label, **low_calibration} save() call(['docker', 'rm', '-f', mock]) if calibration['classification'] != 'pass' or low_calibration['classification'] != 'pass': raise RuntimeError('Generator calibration failed; do not claim API saturation') if args.calibration_only: return for profile in args.profiles: profile_started = datetime.now(timezone.utc).isoformat() n = PROFILE_REPLICAS[profile] print(f'SWITCH {profile}, pool={32//n} per replica', flush=True) env = {**os.environ, 'DB_POOL_MAX': str(32//n)} call(diag.compose(profile)+['up', '-d', '--no-build', '--no-deps', '--wait', '--wait-timeout', '120', 'api'], env=env) time.sleep(6) containers = {role: call(diag.compose(profile)+['ps', '-q', role]).split() for role in ['api', 'postgres', 'nginx', 'redis']} details = json.loads(call(['docker', 'inspect', *[c for group in containers.values() for c in group]])) manifest['containers'][profile] = [{'id': c['Id'], 'name': c['Name'], 'image': c['Image'], 'nano_cpus': c['HostConfig']['NanoCpus'], 'memory_bytes': c['HostConfig']['Memory'], 'pool_max': next((e.split('=', 1)[1] for e in c['Config']['Env'] if e.startswith('DB_POOL_MAX=')), None)} for c in details] profile_dir = out/f'observers-{profile}' profile_dir.mkdir() workers = [] first = len(manifest['runs']) save() try: for role in ['postgres', 'nginx', 'redis']: workers.append(watch(containers[role][0], role, profile_dir)) low = None high = None for rate in ([] if args.confirm_rates else [40, 80, 120, 160, 240, 320, 480]): _, v = run(profile, rate, 'probe', args.probe_seconds, 5, n) if v['classification'] == 'pass': low = rate else: high = rate break if low is None and not args.confirm_rates: for rate in [20, 10]: _, v = run(profile, rate, 'probe', args.probe_seconds, 5, n) if v['classification'] == 'pass': low = rate; break high = rate if low is not None and high is not None: for _ in range(2): if high-low <= 20: break middle = ((low+high)//20)*10 _, v = run(profile, middle, 'probe', args.probe_seconds, 5, n) if v['classification'] == 'pass': low = middle else: high = middle confirmations = {} for repeat in range(3): for rate in sorted(set(args.confirm_rates or [r for r in [low, high] if r is not None])): _, v = run(profile, rate, 'confirmation', args.confirm_seconds, args.warmup_seconds, n) confirmations.setdefault(rate, []).append(v) for _ in range(0 if args.no_expand else 2): good = [r for r, vv in confirmations.items() if all(v['classification'] == 'pass' for v in vv)] bad = [r for r, vv in confirmations.items() if sum(v['classification'] == 'slo_fail' for v in vv) >= 2] if good and bad and max(good) < min(bad): break if any(v['classification'] == 'generator_limited' for vv in confirmations.values() for v in vv): break rate = max(10, min(confirmations)-20) if not good else max(confirmations)+20 if rate in confirmations: break confirmations[rate] = [run(profile, rate, 'confirmation', args.confirm_seconds, args.warmup_seconds, n)[1] for _ in range(3)] good = [r for r, vv in confirmations.items() if all(v['classification'] == 'pass' for v in vv)] bad = [r for r, vv in confirmations.items() if sum(v['classification'] == 'slo_fail' for v in vv) >= 2] manifest['boundaries'][profile] = {'confirmed_good': max(good) if good else None, 'confirmed_bad': min(bad) if bad else None, 'confirmations': confirmations, 'bracketed': bool(good and bad and max(good) < min(bad))} print(f"BOUNDARY {profile}: {manifest['boundaries'][profile]['confirmed_good']} .. " f"{manifest['boundaries'][profile]['confirmed_bad']} RPS", flush=True) finally: for worker in workers: stop_watch(worker, profile_dir) log_result = subprocess.run(['docker', 'logs', '--timestamps', '--since', profile_started, containers['nginx'][0]], cwd=ROOT, capture_output=True, text=True, encoding='utf-8', errors='replace', timeout=30) events = [line for line in (log_result.stdout+'\n'+log_result.stderr).splitlines() if any(term in line for term in ['worker_connections', 'no live upstreams', 'upstream timed out', 'download timed out', 'signal process started'])] (profile_dir/'nginx-events.log').write_text('\n'.join(events)+'\n', encoding='utf-8') for index, container in enumerate(containers['api'], 1): call(['docker', 'cp', f'{container}:/tmp/scaling-pool.jsonl', str(profile_dir/f'api-{index}.jsonl')]) for label in manifest['runs'][first:]: directory = out/label for p in profile_dir.glob('*.jsonl'): (directory/p.name).write_bytes(p.read_bytes()) (directory/'generator.jsonl').write_bytes((out/'generator.jsonl').read_bytes()) b = json.loads((directory/'benchmark.json').read_text()) summary = diag.summarize(directory, b) samples = [r for r in diag.rows(directory/'generator.jsonl') if r['kind'] == 'cgroup' and b['timing']['measurement_started_epoch_ms'] <= r['time_ms'] <= b['timing']['measurement_scheduled_end_epoch_ms']] summary['resources']['generator'] = diag.resource_summary(samples) summary['all_latency_ms'] = b['all_latency_ms'] summary['criterion'] = json.loads((directory/'criterion.json').read_text()) summary['diagnostics_complete'] = len(summary['pools']) == n and all(v['samples'] >= 2 for v in summary['resources'].values()) (directory/'summary.json').write_text(json.dumps(summary, indent=2)+'\n') save() finally: if generator_watch: stop_watch(generator_watch, out) for name in [mock, generator]: subprocess.run(['docker', 'rm', '-f', name], cwd=ROOT, capture_output=True, timeout=30) if not args.calibration_only: print('RESTORE normal two profile, original pool setting', flush=True) call(diag.compose('two', False)+['up', '-d', '--no-build', '--no-deps', '--wait', '--wait-timeout', '120', 'api']) manifest['finished'] = datetime.now(timezone.utc).isoformat() save() if __name__ == '__main__': main()