Files
mtusi2026lab3/scripts/run_experiments.py
T

289 lines
14 KiB
Python

#!/usr/bin/env python3
"""Host-side reproducible experiment controller. Requires Docker Compose v2.
Only this Compose project's API/DB quotas change. No data or volumes are deleted.
The baseline two-replica limits are restored in finally, including after failure.
"""
import argparse
from datetime import datetime, timezone
import hashlib
import json
import os
from pathlib import Path
import subprocess
import threading
import time
from stack_control import containers, docker, snapshot
ROOT = Path(__file__).resolve().parents[1]
def command(args, env=None):
return subprocess.check_output(args, cwd=ROOT, env=env, text=True, encoding='utf-8',
errors='replace', stderr=subprocess.PIPE)
def compose(*args, env=None):
return command(['docker', 'compose', *args], env)
def runner(*args):
return compose('run', '--rm', '-T', 'tester', *args)
def wait_instances(count):
deadline = time.monotonic() + 90
while time.monotonic() < deadline:
apis = containers('api')
if len(apis) == count and all(c['State'].get('Health', {}).get('Status') == 'healthy' for c in apis):
runner('python', 'scripts/wait_ready.py')
# Let Docker DNS/HAProxy rise checks converge after a replacement.
time.sleep(3)
return
time.sleep(.5)
raise RuntimeError('API containers did not become healthy')
def configure(replicas, cpu, memory, db_cpu, pool=4):
env = dict(os.environ, API_REPLICAS=str(replicas), API_CPUS=str(cpu),
API_MEMORY=memory, DB_POOL_SIZE=str(pool))
compose('up', '-d', '--no-deps', '--scale', 'api=' + str(replicas), 'api', env=env)
for c in containers('db'):
docker('update', '--cpus', str(db_cpu), c['Id'])
wait_instances(replicas)
def db_counts():
sql = "SELECT json_build_object('users',(SELECT count(*) FROM users),'chats',(SELECT count(*) FROM chats)," \
"'messages',(SELECT count(*) FROM messages),'sessions',(SELECT count(*) FROM sessions)," \
"'commits',(SELECT xact_commit FROM pg_stat_database WHERE datname=current_database()))"
result = json.loads(compose('exec', '-T', 'db', 'psql', '-U', 'messenger', '-d', 'messenger', '-Atc', sql))
counters = compose('exec', '-T', 'db', 'cat', '/sys/fs/cgroup/cpu.stat')
result['db_cgroup_cpu_stat'] = {k: int(v) for k, v in (line.split() for line in counters.splitlines())}
return result
def monitored_benchmark(name, duration, warmup, concurrency, action=None):
output = 'evidence/scaling/' + name + '.json'
before = db_counts()
samples = []
proc = subprocess.Popen(['docker', 'compose', 'run', '--rm', '-T', 'tester',
'python', os.getenv('BENCHMARK_SCRIPT', 'scripts/benchmark_threads.py'),
'--duration', str(duration), '--warmup', str(warmup),
'--concurrency', str(concurrency), '--output', output], cwd=ROOT,
stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, encoding='utf-8', errors='replace')
action_error = []
def do_action():
try:
time.sleep(warmup + 3)
action()
except Exception as exc:
action_error.append(type(exc).__name__ + ': ' + str(exc))
action_thread = threading.Thread(target=do_action) if action else None
if action_thread:
action_thread.start()
while proc.poll() is None:
try:
ids = [c['Id'] for service in ('api', 'db', 'lb', 'tester') for c in containers(service)]
if ids:
lines = command(['docker', 'stats', '--no-stream', '--format', '{{json .}}', *ids]).splitlines()
samples.append({'utc': datetime.now(timezone.utc).isoformat(),
'containers': [json.loads(line) for line in lines if line]})
except subprocess.CalledProcessError:
# --rm runner or a rolling-update target can disappear between ps,
# inspect and stats. Missing telemetry must not abort a valid run.
samples.append({'utc': datetime.now(timezone.utc).isoformat(), 'containers': [],
'sampling_error': 'Container disappeared or Docker stats unavailable'})
time.sleep(1)
stdout, stderr = proc.communicate()
if action_thread:
action_thread.join()
if proc.returncode or action_error:
raise RuntimeError('Benchmark/action failed: ' + stdout[-1500:] + stderr[-1500:] + str(action_error))
report = json.loads((ROOT / output).read_text(encoding='utf-8'))
meta = {'limits_and_processes': snapshot(), 'dataset_before': before, 'dataset_after': db_counts(),
'resource_samples': samples}
(ROOT / output).with_suffix('.resources.json').write_text(json.dumps(meta, indent=2) + '\n', encoding='utf-8')
print(name, report['successful_rps'], 'RPS; errors', report['failed_requests'], flush=True)
return report
def scaling(args):
dest = ROOT / 'evidence/scaling'
dest.mkdir(parents=True, exist_ok=True)
hardware = {'utc': datetime.now(timezone.utc).isoformat(),
'docker': json.loads(docker('info', '--format', '{{json .}}')),
'note': 'Docker info sanitized to host capacity/version only below'}
info = hardware['docker']
hardware['docker'] = {k: info.get(k) for k in ('NCPU', 'MemTotal', 'Architecture', 'KernelVersion', 'OperatingSystem', 'ServerVersion')}
hardware['images'] = [{ 'image': image, 'id': json.loads(docker('image', 'inspect', image))[0]['Id']}
for image in ('mtusi-lab3-api:local', 'postgres:17.5-bookworm', 'haproxy:3.0.11-alpine')]
hardware['benchmark_script'] = os.getenv('BENCHMARK_SCRIPT', 'scripts/benchmark_threads.py')
(dest / 'hardware.json').write_text(json.dumps(hardware, indent=2) + '\n', encoding='utf-8')
summary = dest / 'summary.json'
records = json.loads(summary.read_text(encoding='utf-8')) if args.resume and summary.exists() else []
completed = {r['name'] for r in records}
# All non-API quotas remain fixed. Fixed: TOTAL API 1CPU/512MiB.
# Growing: EACH API .5CPU/256MiB. DB=.25CPU exposes a shared dependency limit.
cases = [('fixed', n, 1 / n, str(512 // n) + 'm', .25) for n in (1, 2, 4)]
cases += [('growing', n, .5, '256m', .25) for n in (1, 2, 4)]
cases += [('db-improved', 4, .5, '256m', 1.0)]
try:
for series, n, cpu, memory, db_cpu in cases:
if all('%s-n%d-r%d' % (series, n, r) in completed for r in range(1, args.repeats + 1)):
continue
pool = 1 if series == 'pool-one' else 4
configure(n, cpu, memory, db_cpu, pool)
for repeat in range(1, args.repeats + 1):
name = '%s-n%d-r%d' % (series, n, repeat)
if name in completed:
continue
report = monitored_benchmark(name, args.duration, args.warmup, args.concurrency)
records.append({'series': series, 'replicas': n, 'api_cpu_each': cpu,
'api_memory_each': memory, 'db_cpu': db_cpu, 'db_pool_size': pool,
'repeat': repeat, 'name': name, **report})
(dest / 'summary.json').write_text(json.dumps(records, indent=2) + '\n', encoding='utf-8')
# Saturation sweep: identify the concurrency knee with all other limits fixed.
for db_cpu, label in ((.25, 'baseline'), (1.0, 'improved')):
configure(4, .5, '256m', db_cpu)
for concurrency in ((1, 4, 8, 32, 64) if label == 'baseline' else (8, 32, 64)):
for repeat in range(1, args.repeats + 1):
name = 'saturation-%s-c%d-r%d' % (label, concurrency, repeat)
if name in completed:
continue
report = monitored_benchmark(name, args.duration, args.warmup, concurrency)
records.append({'series': 'saturation-' + label, 'replicas': 4,
'api_cpu_each': .5, 'api_memory_each': '256m', 'db_cpu': db_cpu,
'repeat': repeat, 'name': name, **report})
(dest / 'summary.json').write_text(json.dumps(records, indent=2) + '\n', encoding='utf-8')
finally:
compose('up', '-d', '--no-deps', 'lb')
configure(2, .5, '256m', 1.0)
def pool_control(args, records, dest):
# Alternate 4/1/4/1/4/1 instead of comparing two distant best runs.
for repeat in range(1, args.repeats + 1):
for pool, series in ((4, 'pool-four'), (1, 'pool-one')):
configure(4, .5, '256m', .25, pool)
name = '%s-n4-r%d' % (series, repeat)
report = monitored_benchmark(name, args.duration, args.warmup, 32)
records[:] = [r for r in records if r['name'] != name]
records.append({'series': series, 'replicas': 4, 'api_cpu_each': .5,
'api_memory_each': '256m', 'db_cpu': .25, 'db_pool_size': pool,
'repeat': repeat, 'name': name, **report})
(dest / 'summary.json').write_text(json.dumps(records, indent=2) + '\n', encoding='utf-8')
def refine(args):
"""Continue an existing study with lower concurrency and a pool-size control.
Used after the first sweep located its maximum at the smallest tested C=8.
These cases are also included in a fresh full `scaling` run.
"""
dest = ROOT / 'evidence/scaling'
records = json.loads((dest / 'summary.json').read_text(encoding='utf-8'))
cases = [('saturation-baseline', c, 4) for c in (1, 4)]
try:
for series, concurrency, pool in cases:
configure(4, .5, '256m', .25, pool)
for repeat in range(1, args.repeats + 1):
name = ('saturation-baseline-c%d-r%d' % (concurrency, repeat)
if series == 'saturation-baseline' else 'pool-one-n4-r%d' % repeat)
report = monitored_benchmark(name, args.duration, args.warmup, concurrency)
records = [r for r in records if r['name'] != name]
records.append({'series': series, 'replicas': 4, 'api_cpu_each': .5,
'api_memory_each': '256m', 'db_cpu': .25, 'db_pool_size': pool,
'repeat': repeat, 'name': name, **report})
(dest / 'summary.json').write_text(json.dumps(records, indent=2) + '\n', encoding='utf-8')
pool_control(args, records, dest)
finally:
configure(2, .5, '256m', 1.0)
def lb_control(args, records, dest):
configure(4, .5, '256m', .25)
for repeat in range(1, args.repeats + 1):
for bounded, series in ((False, 'lb-unbounded'), (True, 'lb-bounded')):
files = ('-f', 'compose.yaml', '-f', 'infra/compose.lb-limit.yaml') if bounded else ()
compose(*files, 'up', '-d', '--no-deps', 'lb')
runner('python', 'scripts/wait_ready.py')
name = '%s-n4-r%d' % (series, repeat)
report = monitored_benchmark(name, args.duration, args.warmup, 32)
config = ROOT / 'infra' / ('haproxy-bounded.cfg' if bounded else 'haproxy.cfg')
records[:] = [r for r in records if r['name'] != name]
records.append({'series': series, 'replicas': 4, 'api_cpu_each': .5,
'api_memory_each': '256m', 'db_cpu': .25, 'db_pool_size': 4,
'lb_maxconn_per_api': 1 if bounded else 0,
'lb_config_sha256': hashlib.sha256(config.read_bytes()).hexdigest(),
'repeat': repeat, 'name': name, **report})
(dest / 'summary.json').write_text(json.dumps(records, indent=2) + '\n', encoding='utf-8')
text = runner('make', 'test')
(ROOT / 'evidence/bounded-smoke.txt').write_text(text, encoding='utf-8')
def backpressure(args):
dest = ROOT / 'evidence/scaling'
records = json.loads((dest / 'summary.json').read_text(encoding='utf-8'))
try:
lb_control(args, records, dest)
finally:
compose('up', '-d', '--no-deps', 'lb')
configure(2, .5, '256m', 1.0)
def rolling(args):
configure(2, .5, '256m', 1.0)
original = [c['Id'] for c in containers('api')]
events = [{'event': 'before', 'containers': snapshot()}]
def replace():
# Surge +1, then replace originals one at a time, retaining >=2 live APIs.
compose('up', '-d', '--no-deps', '--scale', 'api=3', 'api')
wait_instances(3)
for i, old in enumerate(original):
docker('stop', '--time', '8', old)
docker('rm', old)
compose('up', '-d', '--no-deps', '--scale', 'api=' + str(3 if i == 0 else 2), 'api')
wait_instances(3 if i == 0 else 2)
events.append({'event': 'replaced-' + old[:12], 'utc': datetime.now(timezone.utc).isoformat(),
'containers': snapshot()})
if set(original) & {c['Id'] for c in containers('api')}:
raise RuntimeError('An old API survived replacement')
try:
result = monitored_benchmark('rolling-update', 75, 5, 16, replace)
events.append({'event': 'after', 'containers': snapshot()})
(ROOT / 'evidence/rolling.json').write_text(json.dumps({'events': events, 'load': result,
'raw_prefix': 'evidence/scaling/rolling-update'}, indent=2) + '\n', encoding='utf-8')
finally:
configure(2, .5, '256m', 1.0)
def main():
parser = argparse.ArgumentParser()
parser.add_argument('mode', choices=['scaling', 'refine', 'backpressure', 'rolling'])
parser.add_argument('--duration', type=int, default=20)
parser.add_argument('--warmup', type=int, default=5)
parser.add_argument('--concurrency', type=int, default=32)
parser.add_argument('--repeats', type=int, default=3)
parser.add_argument('--resume', action='store_true', help='Keep completed summary entries and run only missing cases')
args = parser.parse_args()
os.chdir(ROOT)
(ROOT / 'evidence/scaling').mkdir(parents=True, exist_ok=True)
if args.mode == 'scaling':
scaling(args)
elif args.mode == 'refine':
refine(args)
elif args.mode == 'backpressure':
backpressure(args)
else:
rolling(args)
if __name__ == '__main__':
main()