forked from mtusi2026/mtusi2026lab3
289 lines
14 KiB
Python
289 lines
14 KiB
Python
#!/usr/bin/env python3
|
|
"""Host-side reproducible experiment controller. Requires Docker Compose v2.
|
|
|
|
Only this Compose project's API/DB quotas change. No data or volumes are deleted.
|
|
The baseline two-replica limits are restored in finally, including after failure.
|
|
"""
|
|
import argparse
|
|
from datetime import datetime, timezone
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import subprocess
|
|
import threading
|
|
import time
|
|
|
|
from stack_control import containers, docker, snapshot
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
|
|
|
|
def command(args, env=None):
|
|
return subprocess.check_output(args, cwd=ROOT, env=env, text=True, encoding='utf-8',
|
|
errors='replace', stderr=subprocess.PIPE)
|
|
|
|
|
|
def compose(*args, env=None):
|
|
return command(['docker', 'compose', *args], env)
|
|
|
|
|
|
def runner(*args):
|
|
return compose('run', '--rm', '-T', 'tester', *args)
|
|
|
|
|
|
def wait_instances(count):
|
|
deadline = time.monotonic() + 90
|
|
while time.monotonic() < deadline:
|
|
apis = containers('api')
|
|
if len(apis) == count and all(c['State'].get('Health', {}).get('Status') == 'healthy' for c in apis):
|
|
runner('python', 'scripts/wait_ready.py')
|
|
# Let Docker DNS/HAProxy rise checks converge after a replacement.
|
|
time.sleep(3)
|
|
return
|
|
time.sleep(.5)
|
|
raise RuntimeError('API containers did not become healthy')
|
|
|
|
|
|
def configure(replicas, cpu, memory, db_cpu, pool=4):
|
|
env = dict(os.environ, API_REPLICAS=str(replicas), API_CPUS=str(cpu),
|
|
API_MEMORY=memory, DB_POOL_SIZE=str(pool))
|
|
compose('up', '-d', '--no-deps', '--scale', 'api=' + str(replicas), 'api', env=env)
|
|
for c in containers('db'):
|
|
docker('update', '--cpus', str(db_cpu), c['Id'])
|
|
wait_instances(replicas)
|
|
|
|
|
|
def db_counts():
|
|
sql = "SELECT json_build_object('users',(SELECT count(*) FROM users),'chats',(SELECT count(*) FROM chats)," \
|
|
"'messages',(SELECT count(*) FROM messages),'sessions',(SELECT count(*) FROM sessions)," \
|
|
"'commits',(SELECT xact_commit FROM pg_stat_database WHERE datname=current_database()))"
|
|
result = json.loads(compose('exec', '-T', 'db', 'psql', '-U', 'messenger', '-d', 'messenger', '-Atc', sql))
|
|
counters = compose('exec', '-T', 'db', 'cat', '/sys/fs/cgroup/cpu.stat')
|
|
result['db_cgroup_cpu_stat'] = {k: int(v) for k, v in (line.split() for line in counters.splitlines())}
|
|
return result
|
|
|
|
|
|
def monitored_benchmark(name, duration, warmup, concurrency, action=None):
|
|
output = 'evidence/scaling/' + name + '.json'
|
|
before = db_counts()
|
|
samples = []
|
|
proc = subprocess.Popen(['docker', 'compose', 'run', '--rm', '-T', 'tester',
|
|
'python', os.getenv('BENCHMARK_SCRIPT', 'scripts/benchmark_threads.py'),
|
|
'--duration', str(duration), '--warmup', str(warmup),
|
|
'--concurrency', str(concurrency), '--output', output], cwd=ROOT,
|
|
stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, encoding='utf-8', errors='replace')
|
|
action_error = []
|
|
|
|
def do_action():
|
|
try:
|
|
time.sleep(warmup + 3)
|
|
action()
|
|
except Exception as exc:
|
|
action_error.append(type(exc).__name__ + ': ' + str(exc))
|
|
|
|
action_thread = threading.Thread(target=do_action) if action else None
|
|
if action_thread:
|
|
action_thread.start()
|
|
while proc.poll() is None:
|
|
try:
|
|
ids = [c['Id'] for service in ('api', 'db', 'lb', 'tester') for c in containers(service)]
|
|
if ids:
|
|
lines = command(['docker', 'stats', '--no-stream', '--format', '{{json .}}', *ids]).splitlines()
|
|
samples.append({'utc': datetime.now(timezone.utc).isoformat(),
|
|
'containers': [json.loads(line) for line in lines if line]})
|
|
except subprocess.CalledProcessError:
|
|
# --rm runner or a rolling-update target can disappear between ps,
|
|
# inspect and stats. Missing telemetry must not abort a valid run.
|
|
samples.append({'utc': datetime.now(timezone.utc).isoformat(), 'containers': [],
|
|
'sampling_error': 'Container disappeared or Docker stats unavailable'})
|
|
time.sleep(1)
|
|
stdout, stderr = proc.communicate()
|
|
if action_thread:
|
|
action_thread.join()
|
|
if proc.returncode or action_error:
|
|
raise RuntimeError('Benchmark/action failed: ' + stdout[-1500:] + stderr[-1500:] + str(action_error))
|
|
report = json.loads((ROOT / output).read_text(encoding='utf-8'))
|
|
meta = {'limits_and_processes': snapshot(), 'dataset_before': before, 'dataset_after': db_counts(),
|
|
'resource_samples': samples}
|
|
(ROOT / output).with_suffix('.resources.json').write_text(json.dumps(meta, indent=2) + '\n', encoding='utf-8')
|
|
print(name, report['successful_rps'], 'RPS; errors', report['failed_requests'], flush=True)
|
|
return report
|
|
|
|
|
|
def scaling(args):
|
|
dest = ROOT / 'evidence/scaling'
|
|
dest.mkdir(parents=True, exist_ok=True)
|
|
hardware = {'utc': datetime.now(timezone.utc).isoformat(),
|
|
'docker': json.loads(docker('info', '--format', '{{json .}}')),
|
|
'note': 'Docker info sanitized to host capacity/version only below'}
|
|
info = hardware['docker']
|
|
hardware['docker'] = {k: info.get(k) for k in ('NCPU', 'MemTotal', 'Architecture', 'KernelVersion', 'OperatingSystem', 'ServerVersion')}
|
|
hardware['images'] = [{ 'image': image, 'id': json.loads(docker('image', 'inspect', image))[0]['Id']}
|
|
for image in ('mtusi-lab3-api:local', 'postgres:17.5-bookworm', 'haproxy:3.0.11-alpine')]
|
|
hardware['benchmark_script'] = os.getenv('BENCHMARK_SCRIPT', 'scripts/benchmark_threads.py')
|
|
(dest / 'hardware.json').write_text(json.dumps(hardware, indent=2) + '\n', encoding='utf-8')
|
|
summary = dest / 'summary.json'
|
|
records = json.loads(summary.read_text(encoding='utf-8')) if args.resume and summary.exists() else []
|
|
completed = {r['name'] for r in records}
|
|
# All non-API quotas remain fixed. Fixed: TOTAL API 1CPU/512MiB.
|
|
# Growing: EACH API .5CPU/256MiB. DB=.25CPU exposes a shared dependency limit.
|
|
cases = [('fixed', n, 1 / n, str(512 // n) + 'm', .25) for n in (1, 2, 4)]
|
|
cases += [('growing', n, .5, '256m', .25) for n in (1, 2, 4)]
|
|
cases += [('db-improved', 4, .5, '256m', 1.0)]
|
|
try:
|
|
for series, n, cpu, memory, db_cpu in cases:
|
|
if all('%s-n%d-r%d' % (series, n, r) in completed for r in range(1, args.repeats + 1)):
|
|
continue
|
|
pool = 1 if series == 'pool-one' else 4
|
|
configure(n, cpu, memory, db_cpu, pool)
|
|
for repeat in range(1, args.repeats + 1):
|
|
name = '%s-n%d-r%d' % (series, n, repeat)
|
|
if name in completed:
|
|
continue
|
|
report = monitored_benchmark(name, args.duration, args.warmup, args.concurrency)
|
|
records.append({'series': series, 'replicas': n, 'api_cpu_each': cpu,
|
|
'api_memory_each': memory, 'db_cpu': db_cpu, 'db_pool_size': pool,
|
|
'repeat': repeat, 'name': name, **report})
|
|
(dest / 'summary.json').write_text(json.dumps(records, indent=2) + '\n', encoding='utf-8')
|
|
# Saturation sweep: identify the concurrency knee with all other limits fixed.
|
|
for db_cpu, label in ((.25, 'baseline'), (1.0, 'improved')):
|
|
configure(4, .5, '256m', db_cpu)
|
|
for concurrency in ((1, 4, 8, 32, 64) if label == 'baseline' else (8, 32, 64)):
|
|
for repeat in range(1, args.repeats + 1):
|
|
name = 'saturation-%s-c%d-r%d' % (label, concurrency, repeat)
|
|
if name in completed:
|
|
continue
|
|
report = monitored_benchmark(name, args.duration, args.warmup, concurrency)
|
|
records.append({'series': 'saturation-' + label, 'replicas': 4,
|
|
'api_cpu_each': .5, 'api_memory_each': '256m', 'db_cpu': db_cpu,
|
|
'repeat': repeat, 'name': name, **report})
|
|
(dest / 'summary.json').write_text(json.dumps(records, indent=2) + '\n', encoding='utf-8')
|
|
finally:
|
|
compose('up', '-d', '--no-deps', 'lb')
|
|
configure(2, .5, '256m', 1.0)
|
|
|
|
|
|
def pool_control(args, records, dest):
|
|
# Alternate 4/1/4/1/4/1 instead of comparing two distant best runs.
|
|
for repeat in range(1, args.repeats + 1):
|
|
for pool, series in ((4, 'pool-four'), (1, 'pool-one')):
|
|
configure(4, .5, '256m', .25, pool)
|
|
name = '%s-n4-r%d' % (series, repeat)
|
|
report = monitored_benchmark(name, args.duration, args.warmup, 32)
|
|
records[:] = [r for r in records if r['name'] != name]
|
|
records.append({'series': series, 'replicas': 4, 'api_cpu_each': .5,
|
|
'api_memory_each': '256m', 'db_cpu': .25, 'db_pool_size': pool,
|
|
'repeat': repeat, 'name': name, **report})
|
|
(dest / 'summary.json').write_text(json.dumps(records, indent=2) + '\n', encoding='utf-8')
|
|
|
|
|
|
def refine(args):
|
|
"""Continue an existing study with lower concurrency and a pool-size control.
|
|
|
|
Used after the first sweep located its maximum at the smallest tested C=8.
|
|
These cases are also included in a fresh full `scaling` run.
|
|
"""
|
|
dest = ROOT / 'evidence/scaling'
|
|
records = json.loads((dest / 'summary.json').read_text(encoding='utf-8'))
|
|
cases = [('saturation-baseline', c, 4) for c in (1, 4)]
|
|
try:
|
|
for series, concurrency, pool in cases:
|
|
configure(4, .5, '256m', .25, pool)
|
|
for repeat in range(1, args.repeats + 1):
|
|
name = ('saturation-baseline-c%d-r%d' % (concurrency, repeat)
|
|
if series == 'saturation-baseline' else 'pool-one-n4-r%d' % repeat)
|
|
report = monitored_benchmark(name, args.duration, args.warmup, concurrency)
|
|
records = [r for r in records if r['name'] != name]
|
|
records.append({'series': series, 'replicas': 4, 'api_cpu_each': .5,
|
|
'api_memory_each': '256m', 'db_cpu': .25, 'db_pool_size': pool,
|
|
'repeat': repeat, 'name': name, **report})
|
|
(dest / 'summary.json').write_text(json.dumps(records, indent=2) + '\n', encoding='utf-8')
|
|
pool_control(args, records, dest)
|
|
finally:
|
|
configure(2, .5, '256m', 1.0)
|
|
|
|
|
|
def lb_control(args, records, dest):
|
|
configure(4, .5, '256m', .25)
|
|
for repeat in range(1, args.repeats + 1):
|
|
for bounded, series in ((False, 'lb-unbounded'), (True, 'lb-bounded')):
|
|
files = ('-f', 'compose.yaml', '-f', 'infra/compose.lb-limit.yaml') if bounded else ()
|
|
compose(*files, 'up', '-d', '--no-deps', 'lb')
|
|
runner('python', 'scripts/wait_ready.py')
|
|
name = '%s-n4-r%d' % (series, repeat)
|
|
report = monitored_benchmark(name, args.duration, args.warmup, 32)
|
|
config = ROOT / 'infra' / ('haproxy-bounded.cfg' if bounded else 'haproxy.cfg')
|
|
records[:] = [r for r in records if r['name'] != name]
|
|
records.append({'series': series, 'replicas': 4, 'api_cpu_each': .5,
|
|
'api_memory_each': '256m', 'db_cpu': .25, 'db_pool_size': 4,
|
|
'lb_maxconn_per_api': 1 if bounded else 0,
|
|
'lb_config_sha256': hashlib.sha256(config.read_bytes()).hexdigest(),
|
|
'repeat': repeat, 'name': name, **report})
|
|
(dest / 'summary.json').write_text(json.dumps(records, indent=2) + '\n', encoding='utf-8')
|
|
text = runner('make', 'test')
|
|
(ROOT / 'evidence/bounded-smoke.txt').write_text(text, encoding='utf-8')
|
|
|
|
|
|
def backpressure(args):
|
|
dest = ROOT / 'evidence/scaling'
|
|
records = json.loads((dest / 'summary.json').read_text(encoding='utf-8'))
|
|
try:
|
|
lb_control(args, records, dest)
|
|
finally:
|
|
compose('up', '-d', '--no-deps', 'lb')
|
|
configure(2, .5, '256m', 1.0)
|
|
|
|
|
|
def rolling(args):
|
|
configure(2, .5, '256m', 1.0)
|
|
original = [c['Id'] for c in containers('api')]
|
|
events = [{'event': 'before', 'containers': snapshot()}]
|
|
|
|
def replace():
|
|
# Surge +1, then replace originals one at a time, retaining >=2 live APIs.
|
|
compose('up', '-d', '--no-deps', '--scale', 'api=3', 'api')
|
|
wait_instances(3)
|
|
for i, old in enumerate(original):
|
|
docker('stop', '--time', '8', old)
|
|
docker('rm', old)
|
|
compose('up', '-d', '--no-deps', '--scale', 'api=' + str(3 if i == 0 else 2), 'api')
|
|
wait_instances(3 if i == 0 else 2)
|
|
events.append({'event': 'replaced-' + old[:12], 'utc': datetime.now(timezone.utc).isoformat(),
|
|
'containers': snapshot()})
|
|
if set(original) & {c['Id'] for c in containers('api')}:
|
|
raise RuntimeError('An old API survived replacement')
|
|
|
|
try:
|
|
result = monitored_benchmark('rolling-update', 75, 5, 16, replace)
|
|
events.append({'event': 'after', 'containers': snapshot()})
|
|
(ROOT / 'evidence/rolling.json').write_text(json.dumps({'events': events, 'load': result,
|
|
'raw_prefix': 'evidence/scaling/rolling-update'}, indent=2) + '\n', encoding='utf-8')
|
|
finally:
|
|
configure(2, .5, '256m', 1.0)
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument('mode', choices=['scaling', 'refine', 'backpressure', 'rolling'])
|
|
parser.add_argument('--duration', type=int, default=20)
|
|
parser.add_argument('--warmup', type=int, default=5)
|
|
parser.add_argument('--concurrency', type=int, default=32)
|
|
parser.add_argument('--repeats', type=int, default=3)
|
|
parser.add_argument('--resume', action='store_true', help='Keep completed summary entries and run only missing cases')
|
|
args = parser.parse_args()
|
|
os.chdir(ROOT)
|
|
(ROOT / 'evidence/scaling').mkdir(parents=True, exist_ok=True)
|
|
if args.mode == 'scaling':
|
|
scaling(args)
|
|
elif args.mode == 'refine':
|
|
refine(args)
|
|
elif args.mode == 'backpressure':
|
|
backpressure(args)
|
|
else:
|
|
rolling(args)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|