#!/usr/bin/env python3 """Host-side reproducible experiment controller. Requires Docker Compose v2. Only this Compose project's API/DB quotas change. No data or volumes are deleted. The baseline two-replica limits are restored in finally, including after failure. """ import argparse from datetime import datetime, timezone import hashlib import json import os from pathlib import Path import subprocess import threading import time from stack_control import containers, docker, snapshot ROOT = Path(__file__).resolve().parents[1] def command(args, env=None): return subprocess.check_output(args, cwd=ROOT, env=env, text=True, encoding='utf-8', errors='replace', stderr=subprocess.PIPE) def compose(*args, env=None): return command(['docker', 'compose', *args], env) def runner(*args): return compose('run', '--rm', '-T', 'tester', *args) def wait_instances(count): deadline = time.monotonic() + 90 while time.monotonic() < deadline: apis = containers('api') if len(apis) == count and all(c['State'].get('Health', {}).get('Status') == 'healthy' for c in apis): runner('python', 'scripts/wait_ready.py') # Let Docker DNS/HAProxy rise checks converge after a replacement. time.sleep(3) return time.sleep(.5) raise RuntimeError('API containers did not become healthy') def configure(replicas, cpu, memory, db_cpu, pool=4): env = dict(os.environ, API_REPLICAS=str(replicas), API_CPUS=str(cpu), API_MEMORY=memory, DB_POOL_SIZE=str(pool)) compose('up', '-d', '--no-deps', '--scale', 'api=' + str(replicas), 'api', env=env) for c in containers('db'): docker('update', '--cpus', str(db_cpu), c['Id']) wait_instances(replicas) def db_counts(): sql = "SELECT json_build_object('users',(SELECT count(*) FROM users),'chats',(SELECT count(*) FROM chats)," \ "'messages',(SELECT count(*) FROM messages),'sessions',(SELECT count(*) FROM sessions)," \ "'commits',(SELECT xact_commit FROM pg_stat_database WHERE datname=current_database()))" result = json.loads(compose('exec', '-T', 'db', 'psql', '-U', 'messenger', '-d', 'messenger', '-Atc', sql)) counters = compose('exec', '-T', 'db', 'cat', '/sys/fs/cgroup/cpu.stat') result['db_cgroup_cpu_stat'] = {k: int(v) for k, v in (line.split() for line in counters.splitlines())} return result def monitored_benchmark(name, duration, warmup, concurrency, action=None): output = 'evidence/scaling/' + name + '.json' before = db_counts() samples = [] proc = subprocess.Popen(['docker', 'compose', 'run', '--rm', '-T', 'tester', 'python', os.getenv('BENCHMARK_SCRIPT', 'scripts/benchmark_threads.py'), '--duration', str(duration), '--warmup', str(warmup), '--concurrency', str(concurrency), '--output', output], cwd=ROOT, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, encoding='utf-8', errors='replace') action_error = [] def do_action(): try: time.sleep(warmup + 3) action() except Exception as exc: action_error.append(type(exc).__name__ + ': ' + str(exc)) action_thread = threading.Thread(target=do_action) if action else None if action_thread: action_thread.start() while proc.poll() is None: try: ids = [c['Id'] for service in ('api', 'db', 'lb', 'tester') for c in containers(service)] if ids: lines = command(['docker', 'stats', '--no-stream', '--format', '{{json .}}', *ids]).splitlines() samples.append({'utc': datetime.now(timezone.utc).isoformat(), 'containers': [json.loads(line) for line in lines if line]}) except subprocess.CalledProcessError: # --rm runner or a rolling-update target can disappear between ps, # inspect and stats. Missing telemetry must not abort a valid run. samples.append({'utc': datetime.now(timezone.utc).isoformat(), 'containers': [], 'sampling_error': 'Container disappeared or Docker stats unavailable'}) time.sleep(1) stdout, stderr = proc.communicate() if action_thread: action_thread.join() if proc.returncode or action_error: raise RuntimeError('Benchmark/action failed: ' + stdout[-1500:] + stderr[-1500:] + str(action_error)) report = json.loads((ROOT / output).read_text(encoding='utf-8')) meta = {'limits_and_processes': snapshot(), 'dataset_before': before, 'dataset_after': db_counts(), 'resource_samples': samples} (ROOT / output).with_suffix('.resources.json').write_text(json.dumps(meta, indent=2) + '\n', encoding='utf-8') print(name, report['successful_rps'], 'RPS; errors', report['failed_requests'], flush=True) return report def scaling(args): dest = ROOT / 'evidence/scaling' dest.mkdir(parents=True, exist_ok=True) hardware = {'utc': datetime.now(timezone.utc).isoformat(), 'docker': json.loads(docker('info', '--format', '{{json .}}')), 'note': 'Docker info sanitized to host capacity/version only below'} info = hardware['docker'] hardware['docker'] = {k: info.get(k) for k in ('NCPU', 'MemTotal', 'Architecture', 'KernelVersion', 'OperatingSystem', 'ServerVersion')} hardware['images'] = [{ 'image': image, 'id': json.loads(docker('image', 'inspect', image))[0]['Id']} for image in ('mtusi-lab3-api:local', 'postgres:17.5-bookworm', 'haproxy:3.0.11-alpine')] hardware['benchmark_script'] = os.getenv('BENCHMARK_SCRIPT', 'scripts/benchmark_threads.py') (dest / 'hardware.json').write_text(json.dumps(hardware, indent=2) + '\n', encoding='utf-8') summary = dest / 'summary.json' records = json.loads(summary.read_text(encoding='utf-8')) if args.resume and summary.exists() else [] completed = {r['name'] for r in records} # All non-API quotas remain fixed. Fixed: TOTAL API 1CPU/512MiB. # Growing: EACH API .5CPU/256MiB. DB=.25CPU exposes a shared dependency limit. cases = [('fixed', n, 1 / n, str(512 // n) + 'm', .25) for n in (1, 2, 4)] cases += [('growing', n, .5, '256m', .25) for n in (1, 2, 4)] cases += [('db-improved', 4, .5, '256m', 1.0)] try: for series, n, cpu, memory, db_cpu in cases: if all('%s-n%d-r%d' % (series, n, r) in completed for r in range(1, args.repeats + 1)): continue pool = 1 if series == 'pool-one' else 4 configure(n, cpu, memory, db_cpu, pool) for repeat in range(1, args.repeats + 1): name = '%s-n%d-r%d' % (series, n, repeat) if name in completed: continue report = monitored_benchmark(name, args.duration, args.warmup, args.concurrency) records.append({'series': series, 'replicas': n, 'api_cpu_each': cpu, 'api_memory_each': memory, 'db_cpu': db_cpu, 'db_pool_size': pool, 'repeat': repeat, 'name': name, **report}) (dest / 'summary.json').write_text(json.dumps(records, indent=2) + '\n', encoding='utf-8') # Saturation sweep: identify the concurrency knee with all other limits fixed. for db_cpu, label in ((.25, 'baseline'), (1.0, 'improved')): configure(4, .5, '256m', db_cpu) for concurrency in ((1, 4, 8, 32, 64) if label == 'baseline' else (8, 32, 64)): for repeat in range(1, args.repeats + 1): name = 'saturation-%s-c%d-r%d' % (label, concurrency, repeat) if name in completed: continue report = monitored_benchmark(name, args.duration, args.warmup, concurrency) records.append({'series': 'saturation-' + label, 'replicas': 4, 'api_cpu_each': .5, 'api_memory_each': '256m', 'db_cpu': db_cpu, 'repeat': repeat, 'name': name, **report}) (dest / 'summary.json').write_text(json.dumps(records, indent=2) + '\n', encoding='utf-8') finally: compose('up', '-d', '--no-deps', 'lb') configure(2, .5, '256m', 1.0) def pool_control(args, records, dest): # Alternate 4/1/4/1/4/1 instead of comparing two distant best runs. for repeat in range(1, args.repeats + 1): for pool, series in ((4, 'pool-four'), (1, 'pool-one')): configure(4, .5, '256m', .25, pool) name = '%s-n4-r%d' % (series, repeat) report = monitored_benchmark(name, args.duration, args.warmup, 32) records[:] = [r for r in records if r['name'] != name] records.append({'series': series, 'replicas': 4, 'api_cpu_each': .5, 'api_memory_each': '256m', 'db_cpu': .25, 'db_pool_size': pool, 'repeat': repeat, 'name': name, **report}) (dest / 'summary.json').write_text(json.dumps(records, indent=2) + '\n', encoding='utf-8') def refine(args): """Continue an existing study with lower concurrency and a pool-size control. Used after the first sweep located its maximum at the smallest tested C=8. These cases are also included in a fresh full `scaling` run. """ dest = ROOT / 'evidence/scaling' records = json.loads((dest / 'summary.json').read_text(encoding='utf-8')) cases = [('saturation-baseline', c, 4) for c in (1, 4)] try: for series, concurrency, pool in cases: configure(4, .5, '256m', .25, pool) for repeat in range(1, args.repeats + 1): name = ('saturation-baseline-c%d-r%d' % (concurrency, repeat) if series == 'saturation-baseline' else 'pool-one-n4-r%d' % repeat) report = monitored_benchmark(name, args.duration, args.warmup, concurrency) records = [r for r in records if r['name'] != name] records.append({'series': series, 'replicas': 4, 'api_cpu_each': .5, 'api_memory_each': '256m', 'db_cpu': .25, 'db_pool_size': pool, 'repeat': repeat, 'name': name, **report}) (dest / 'summary.json').write_text(json.dumps(records, indent=2) + '\n', encoding='utf-8') pool_control(args, records, dest) finally: configure(2, .5, '256m', 1.0) def lb_control(args, records, dest): configure(4, .5, '256m', .25) for repeat in range(1, args.repeats + 1): for bounded, series in ((False, 'lb-unbounded'), (True, 'lb-bounded')): files = ('-f', 'compose.yaml', '-f', 'infra/compose.lb-limit.yaml') if bounded else () compose(*files, 'up', '-d', '--no-deps', 'lb') runner('python', 'scripts/wait_ready.py') name = '%s-n4-r%d' % (series, repeat) report = monitored_benchmark(name, args.duration, args.warmup, 32) config = ROOT / 'infra' / ('haproxy-bounded.cfg' if bounded else 'haproxy.cfg') records[:] = [r for r in records if r['name'] != name] records.append({'series': series, 'replicas': 4, 'api_cpu_each': .5, 'api_memory_each': '256m', 'db_cpu': .25, 'db_pool_size': 4, 'lb_maxconn_per_api': 1 if bounded else 0, 'lb_config_sha256': hashlib.sha256(config.read_bytes()).hexdigest(), 'repeat': repeat, 'name': name, **report}) (dest / 'summary.json').write_text(json.dumps(records, indent=2) + '\n', encoding='utf-8') text = runner('make', 'test') (ROOT / 'evidence/bounded-smoke.txt').write_text(text, encoding='utf-8') def backpressure(args): dest = ROOT / 'evidence/scaling' records = json.loads((dest / 'summary.json').read_text(encoding='utf-8')) try: lb_control(args, records, dest) finally: compose('up', '-d', '--no-deps', 'lb') configure(2, .5, '256m', 1.0) def rolling(args): configure(2, .5, '256m', 1.0) original = [c['Id'] for c in containers('api')] events = [{'event': 'before', 'containers': snapshot()}] def replace(): # Surge +1, then replace originals one at a time, retaining >=2 live APIs. compose('up', '-d', '--no-deps', '--scale', 'api=3', 'api') wait_instances(3) for i, old in enumerate(original): docker('stop', '--time', '8', old) docker('rm', old) compose('up', '-d', '--no-deps', '--scale', 'api=' + str(3 if i == 0 else 2), 'api') wait_instances(3 if i == 0 else 2) events.append({'event': 'replaced-' + old[:12], 'utc': datetime.now(timezone.utc).isoformat(), 'containers': snapshot()}) if set(original) & {c['Id'] for c in containers('api')}: raise RuntimeError('An old API survived replacement') try: result = monitored_benchmark('rolling-update', 75, 5, 16, replace) events.append({'event': 'after', 'containers': snapshot()}) (ROOT / 'evidence/rolling.json').write_text(json.dumps({'events': events, 'load': result, 'raw_prefix': 'evidence/scaling/rolling-update'}, indent=2) + '\n', encoding='utf-8') finally: configure(2, .5, '256m', 1.0) def main(): parser = argparse.ArgumentParser() parser.add_argument('mode', choices=['scaling', 'refine', 'backpressure', 'rolling']) parser.add_argument('--duration', type=int, default=20) parser.add_argument('--warmup', type=int, default=5) parser.add_argument('--concurrency', type=int, default=32) parser.add_argument('--repeats', type=int, default=3) parser.add_argument('--resume', action='store_true', help='Keep completed summary entries and run only missing cases') args = parser.parse_args() os.chdir(ROOT) (ROOT / 'evidence/scaling').mkdir(parents=True, exist_ok=True) if args.mode == 'scaling': scaling(args) elif args.mode == 'refine': refine(args) elif args.mode == 'backpressure': backpressure(args) else: rolling(args) if __name__ == '__main__': main()