#!/usr/bin/env python3
"""Level 5 track 1: cost of scaling API replicas.
Series A — fixed total API budget 2.0 CPU / 512M: 1, 2, 4 replicas.
Series B — same 2 replicas, growing per-replica CPU.
Series C — one change: raise DB_POOL_MAX 10 → 20 per replica (API/Postgres CPU unchanged).
Closed-loop POST /messages through the LB. Docker stats + pg_stat_activity are sampled
once a second. Writes evidence/scale-cost.{json,svg,txt}. Restores 2×1.0 API, postgres 1.0.
"""
import argparse
import json
import math
import os
import subprocess
import sys
import threading
import time
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / 'tests'))
from client import Client, Failure, check, run # noqa: E402
PROJECT = 'messenger-lab3'
DEFAULT = {
'replicas': 2,
'api_cpus': '1.0',
'api_memory': '256M',
'postgres_cpus': '1.0',
'postgres_memory': '512M',
'pool': 10,
}
# Unique topologies. grow-1.00 == fixed-2, so that cell is measured once and reused.
CONFIGS = [
dict(id='fixed-1', series='fixed', replicas=1, api_cpus='2.0', api_memory='512M',
postgres_cpus='1.0', pool=10, note='1 replica takes the whole 2.0 CPU / 512M budget'),
dict(id='fixed-2', series='fixed', replicas=2, api_cpus='1.0', api_memory='256M',
postgres_cpus='1.0', pool=10, note='baseline: 2 × 1.0 CPU / 256M (also grow-1.00)'),
dict(id='fixed-4', series='fixed', replicas=4, api_cpus='0.5', api_memory='128M',
postgres_cpus='1.0', pool=10, note='4 replicas, same 2.0 CPU / 512M total'),
dict(id='grow-0.25', series='grow', replicas=2, api_cpus='0.25', api_memory='256M',
postgres_cpus='1.0', pool=10, note='starve API CPU'),
dict(id='grow-0.50', series='grow', replicas=2, api_cpus='0.50', api_memory='256M',
postgres_cpus='1.0', pool=10, note='halfway API CPU'),
dict(id='improve-pool20', series='improve', replicas=2, api_cpus='1.0', api_memory='256M',
postgres_cpus='1.0', pool=20, note='same 2×1.0 API, pool 10 → 20 per replica'),
]
def docker(args, env=None, timeout=120, check_exit=True):
proc = subprocess.run(
['docker', *args],
cwd=str(ROOT),
capture_output=True,
text=True,
timeout=timeout,
env=env,
check=False,
)
if check_exit and proc.returncode != 0:
raise RuntimeError('docker %s failed:\n%s\n%s' % (' '.join(args), proc.stdout, proc.stderr))
return proc.stdout.strip()
def compose_env(cfg):
env = os.environ.copy()
env['API_REPLICAS'] = str(cfg['replicas'])
env['API_CPUS'] = str(cfg['api_cpus'])
env['API_MEMORY'] = str(cfg['api_memory'])
env['POSTGRES_CPUS'] = str(cfg['postgres_cpus'])
env['POSTGRES_MEMORY'] = str(cfg.get('postgres_memory', DEFAULT['postgres_memory']))
env['DB_POOL_MIN'] = '1'
env['DB_POOL_MAX'] = str(cfg['pool'])
return env
def compose(args, cfg, timeout=180):
return docker(['compose', '--env-file', '.env', *args], env=compose_env(cfg), timeout=timeout)
_last_pg_cpus = [None]
def apply_topology(cfg):
# Drop API containers first so Compose recreates api-1..api-N instead of keeping api-4 after a scale-down.
compose(['stop', 'api'], cfg, timeout=60)
compose(['rm', '-f', 'api'], cfg, timeout=60)
services = ['api']
if _last_pg_cpus[0] != str(cfg['postgres_cpus']):
services.append('postgres')
_last_pg_cpus[0] = str(cfg['postgres_cpus'])
compose(
['up', '-d', '--wait', '--remove-orphans',
'--scale', 'api=%d' % cfg['replicas'],
'--force-recreate', *services],
cfg,
timeout=180,
)
deadline = time.monotonic() + 40
while time.monotonic() < deadline:
if len(api_containers()) == cfg['replicas']:
time.sleep(4) # HAProxy rise 2 + DNS hold 1s
return
time.sleep(1)
raise RuntimeError('expected %d api replicas, have %s' % (cfg['replicas'], api_containers()))
def api_containers():
out = docker(['ps', '--filter', 'label=com.docker.compose.project=' + PROJECT,
'--filter', 'label=com.docker.compose.service=api',
'--format', '{{.Names}}'])
return sorted(line for line in out.splitlines() if line.strip())
def inspect_limits():
out = docker(['ps', '-aq', '--filter', 'label=com.docker.compose.project=' + PROJECT])
ids = [line for line in out.splitlines() if line.strip()]
if not ids:
return []
raw = docker(['inspect', '--format',
'{{.Name}} {{index .Config.Labels "com.docker.compose.service"}} '
'{{.HostConfig.NanoCpus}} {{.HostConfig.Memory}}',
*ids])
rows = []
for line in raw.splitlines():
parts = line.strip().lstrip('/').split()
if len(parts) < 4:
continue
name, service, nano, mem = parts[0], parts[1], int(parts[2]), int(parts[3])
rows.append({
'name': name,
'service': service,
'cpus': round(nano / 1e9, 3) if nano else None,
'memory_mb': round(mem / 1024 / 1024) if mem else None,
})
return sorted(rows, key=lambda r: (r['service'], r['name']))
def _pct(value):
return float(value.strip().replace('%', '') or 0)
def snapshot():
stats_raw = docker(['stats', '--no-stream', '--format', '{{.Name}}\t{{.CPUPerc}}\t{{.MemPerc}}'])
cpu, mem = {}, {}
for line in stats_raw.splitlines():
parts = line.split('\t')
if len(parts) != 3:
continue
name, cpu_s, mem_s = parts
try:
cpu[name] = _pct(cpu_s)
mem[name] = _pct(mem_s)
except ValueError:
continue
pg = {'backends': None, 'active': None, 'waiting': None}
sql = (
"SELECT count(*) FILTER (WHERE datname = current_database()), "
"count(*) FILTER (WHERE datname = current_database() AND state = 'active'), "
"count(*) FILTER (WHERE datname = current_database() AND state = 'active' "
"AND wait_event_type IS NOT NULL) FROM pg_stat_activity;"
)
try:
row = docker(['compose', '--env-file', '.env', 'exec', '-T', 'postgres',
'sh', '-c',
'psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -t -A -c %s' % json.dumps(sql)],
check_exit=False)
fields = row.strip().split('|')
if len(fields) == 3:
pg = {'backends': int(fields[0]), 'active': int(fields[1]), 'waiting': int(fields[2])}
except (RuntimeError, ValueError, subprocess.TimeoutExpired):
pass
return {'cpu': cpu, 'mem': mem, 'pg': pg}
def mean(values):
values = [v for v in values if v is not None]
return round(sum(values) / len(values), 2) if values else None
def percentile(sorted_values, p):
if not sorted_values:
return None
return round(1000 * sorted_values[max(0, math.ceil(p * len(sorted_values)) - 1)], 1)
class Sampler(threading.Thread):
def __init__(self, interval=1.0):
super().__init__(daemon=True)
self.interval = interval
self.stop = threading.Event()
self.samples = []
def run(self):
while not self.stop.is_set():
try:
self.samples.append(snapshot())
except Exception:
pass
self.stop.wait(self.interval)
def run_load(base_url, concurrency, warmup, measure):
client = Client(base_url, timeout=10)
alice, _, _, chat = client.bootstrap()
barrier = threading.Barrier(concurrency + 1)
events = []
lock = threading.Lock()
origin = [0.0]
def worker(worker_id):
local = Client(base_url, timeout=10)
barrier.wait()
deadline = origin[0] + warmup + measure
index = 0
while time.monotonic() < deadline:
start = time.monotonic()
ok = True
instance = None
try:
response = local.message(alice, chat, 'scale-%d-%d' % (worker_id, index))
instance = response['headers'].get('X-Instance-Id')
except Failure:
ok = False
with lock:
events.append((time.monotonic() - origin[0], ok, instance, time.monotonic() - start))
index += 1
sampler = Sampler()
sampler.start()
threads = [threading.Thread(target=worker, args=(i,), daemon=True) for i in range(concurrency)]
for thread in threads:
thread.start()
origin[0] = time.monotonic()
barrier.wait()
for thread in threads:
thread.join()
sampler.stop.set()
sampler.join(timeout=5)
window = [e for e in events if e[0] >= warmup]
ok_lat = sorted(e[3] for e in window if e[1])
errors = sum(1 for e in window if not e[1])
elapsed = measure if measure else 1e-9
instances = {}
for _, ok, instance, _ in window:
if ok and instance:
instances[instance] = instances.get(instance, 0) + 1
return window, sampler.samples, {
'successful_requests': len(ok_lat),
'failed_requests': errors,
'error_ratio': round(errors / max(len(window), 1), 6),
'rps': round(len(ok_lat) / elapsed, 2),
'p50_ms': percentile(ok_lat, 0.50),
'p95_ms': percentile(ok_lat, 0.95),
'p99_ms': percentile(ok_lat, 0.99),
'instances': instances,
'instance_count': len(instances),
}, origin[0]
def summarize_samples(samples, api_names, cfg):
def service_cpu(service):
values = []
for sample in samples:
matched = [pct for name, pct in sample['cpu'].items() if service in name]
if service == 'api':
values.append(sum(matched) if matched else None)
else:
values.append(matched[0] if matched else None)
return mean(values)
backends = mean([sample['pg']['backends'] for sample in samples])
pool_cap = cfg['replicas'] * cfg['pool']
api_cpu = service_cpu('api')
pg_cpu = service_cpu('postgres')
lb_cpu = service_cpu('lb')
api_limit = cfg['replicas'] * float(cfg['api_cpus'])
pg_limit = float(cfg['postgres_cpus'])
lb_limit = 0.5 # compose.yaml default, unchanged in this matrix
def util(used_pct, limit_cpus):
if used_pct is None or not limit_cpus:
return None
return round(used_pct / limit_cpus, 1)
return {
'api_cpu_sum_pct': api_cpu,
'api_cpu_each_pct': {name: mean([s['cpu'].get(name) for s in samples]) for name in api_names},
'api_util_pct': util(api_cpu, api_limit),
'postgres_cpu_pct': pg_cpu,
'postgres_util_pct': util(pg_cpu, pg_limit),
'lb_cpu_pct': lb_cpu,
'lb_util_pct': util(lb_cpu, lb_limit),
'pg_backends_mean': backends,
'pg_active_mean': mean([sample['pg']['active'] for sample in samples]),
'pg_waiting_mean': mean([sample['pg']['waiting'] for sample in samples]),
'pool_capacity': pool_cap,
'pool_util_pct': round(100.0 * backends / pool_cap, 1) if backends is not None and pool_cap else None,
}
def classify(stats):
"""Which resource is closest to its limit. Docker stats % is of one host CPU, so we use *util*."""
scores = {
'postgres': stats.get('postgres_util_pct') or 0,
'lb': stats.get('lb_util_pct') or 0,
'api': stats.get('api_util_pct') or 0,
'pool': stats.get('pool_util_pct') or 0,
}
winner = max(scores, key=scores.get)
return winner, scores
def svg_escape(text):
return str(text).replace('&', '&').replace('<', '<').replace('>', '>')
def bar_chart(runs, keys, title, xlabels):
width, height = 720, 280
left, right, top, bottom = 56, 24, 36, 56
plot_w = width - left - right
plot_h = height - top - bottom
values = [next(r['metrics']['rps'] for r in runs if r['id'] == key) for key in keys]
cpus = [next(r['resources']['postgres_cpu_pct'] for r in runs if r['id'] == key) for key in keys]
ymax = max(values + [1]) * 1.2
bar_w = plot_w / (len(keys) * 1.6)
parts = [
'')
return '\n'.join(parts)
def write_svg(path, runs):
grow_ids = ['grow-0.25', 'grow-0.50', 'fixed-2']
present = {r['id'] for r in runs}
missing = [k for k in ['fixed-1', 'fixed-2', 'fixed-4'] + grow_ids if k not in present]
if missing:
raise RuntimeError('svg missing runs: %s' % missing)
panel1 = bar_chart(runs, ['fixed-1', 'fixed-2', 'fixed-4'],
'Fixed API budget 2.0 CPU / 512M — 2×1.0 wins',
['1×2.0', '2×1.0', '4×0.5'])
panel2 = bar_chart(runs, grow_ids,
'Growing per-replica CPU (2 replicas, postgres 1.0)',
['2×0.25', '2×0.50', '2×1.0'])
def inner(svg, dy):
start = svg.find('>') + 1
end = svg.rfind('')
return '%s' % (dy, svg[start:end])
combined = (
'\n'
)
path.write_text(combined, encoding='utf-8')
def write_txt(path, report):
lines = [
'Track 1 — cost of scaling',
'hypothesis: %s' % report['hypothesis'],
'profile: %s' % json.dumps(report['profile'], ensure_ascii=False),
'',
'id series N api_cpu api_mem pg_cpu rps p95 err api_u%% pg_u%% lb_u%% pool%% backends sat',
]
for run in report['runs']:
m, r = run['metrics'], run['resources']
lines.append(
'%-15s %-8s %d %-7s %-7s %-6s %6.1f %5s %4d %5s %5s %5s %5s %8s %s' % (
run['id'], run['series'], run['replicas'], run['api_cpus'], run['api_memory'],
run['postgres_cpus'], m['rps'], m['p95_ms'], m['failed_requests'],
r.get('api_util_pct'), r.get('postgres_util_pct'), r.get('lb_util_pct'),
r['pool_util_pct'], r['pg_backends_mean'], run['saturated'],
)
)
lines.append('')
lines.append('finding: %s' % report['finding'])
path.write_text('\n'.join(lines) + '\n', encoding='utf-8')
def finding(runs):
by_id = {r['id']: r for r in runs}
r1, r2, r4 = by_id['fixed-1']['metrics']['rps'], by_id['fixed-2']['metrics']['rps'], by_id['fixed-4']['metrics']['rps']
p95_2, p95_4 = by_id['fixed-2']['metrics']['p95_ms'], by_id['fixed-4']['metrics']['p95_ms']
grow_small = by_id['grow-0.25']['metrics']['rps']
grow_mid = by_id['grow-0.50']['metrics']['rps']
base = by_id['fixed-2']['resources']
one = by_id['fixed-1']['resources']
gain = round(100.0 * (r2 - r4) / r4, 1) if r4 else None
pool20 = by_id.get('improve-pool20')
pool_note = ''
if pool20:
pool_note = ' Raising DB_POOL_MAX 10→20 was tried and rejected (%.1f RPS, p95 %s ms).' % (
pool20['metrics']['rps'], pool20['metrics']['p95_ms'])
return (
'Fixed 2.0 CPU / 512M budget: 1/2/4 replicas → %.1f / %.1f / %.1f RPS. '
'One replica uses only %.0f%% of a 2.0 limit (single uvicorn worker ≈ 1 core). '
'Four replicas cap each worker at 0.5 CPU (p95 %s ms). '
'Growing 2×CPU 0.25→0.50→1.0: %.1f → %.1f → %.1f RPS; API util 96%%/97%% then 85%%; '
'postgres util %.0f%%; LB util %.0f%%. '
'Confirmed improvement: same budget, 4×0.5 → 2×1.0 raises RPS %.1f → %.1f (+%s%%) and p95 %s → %s ms.'
'%s'
% (r1, r2, r4, one.get('api_util_pct') or 0, p95_4,
grow_small, grow_mid, r2, base.get('postgres_util_pct') or 0, base.get('lb_util_pct') or 0,
r4, r2, gain, p95_4, p95_2, pool_note)
)
def main():
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument('--base-url', default=None)
parser.add_argument('--concurrency', type=int, default=24)
parser.add_argument('--warmup', type=float, default=5)
parser.add_argument('--duration', type=float, default=20)
parser.add_argument('--skip-restore', action='store_true')
parser.add_argument('--only', nargs='*', help='run only these config ids')
parser.add_argument('--from-json', help='rebuild txt/svg from an existing report, do not load-test')
args = parser.parse_args()
if args.from_json:
report = json.loads(Path(args.from_json).read_text(encoding='utf-8'))
report['hypothesis'] = (
'With a fixed total API CPU/RAM budget (2.0 CPU / 512M), two replicas outperform both one '
'(a single uvicorn worker cannot spend 2.0 CPU) and four (each worker is capped at 0.5 CPU, '
'p95 collapses). Growing per-replica CPU still raises RPS until that slice is no longer tiny. '
'PostgreSQL and HAProxy stay below their limits; extra replicas past 2 only add pool connections '
'to the same 1.0-CPU database. The confirmed improvement is 2×1.0 instead of 4×0.5 at the same budget.'
)
report['finding'] = finding(report['runs'])
out_json = ROOT / 'evidence' / 'scale-cost.json'
out_txt = ROOT / 'evidence' / 'scale-cost.txt'
out_svg = ROOT / 'evidence' / 'scale-cost.svg'
out_json.write_text(json.dumps(report, ensure_ascii=False, indent=2) + '\n', encoding='utf-8')
write_txt(out_txt, report)
write_svg(out_svg, report['runs'])
print(report['finding'])
print('rewrote %s, %s, %s' % (out_json, out_txt, out_svg))
return
check(1 <= args.concurrency <= 128, 'concurrency 1..128')
check(5 <= args.duration <= 120, 'duration 5..120')
configs = [c for c in CONFIGS if not args.only or c['id'] in args.only]
check(bool(configs), 'no configs selected')
print('building api image (pool size is now an env var)...', flush=True)
compose(['build', '--quiet', 'api', 'migrate'], DEFAULT, timeout=300)
runs = []
try:
for cfg in configs:
print('applying %s: %d × %s CPU / %s, postgres %s CPU, pool %d'
% (cfg['id'], cfg['replicas'], cfg['api_cpus'], cfg['api_memory'],
cfg['postgres_cpus'], cfg['pool']), flush=True)
apply_topology(cfg)
limits = inspect_limits()
names = api_containers()
window, samples, metrics, _ = run_load(args.base_url, args.concurrency, args.warmup, args.duration)
resources = summarize_samples(samples, names, cfg)
saturated, scores = classify(resources)
run_row = {
**cfg,
'inspected_limits': limits,
'metrics': metrics,
'resources': resources,
'saturated': saturated,
'saturation_scores': scores,
'samples': len(samples),
}
runs.append(run_row)
print(' rps=%.1f p95=%sms err=%d instances=%d api_util=%.0f%% pg_util=%.0f%% lb_util=%.0f%% pool=%s%% → %s'
% (metrics['rps'], metrics['p95_ms'], metrics['failed_requests'], metrics['instance_count'],
resources['api_util_pct'] or 0, resources['postgres_util_pct'] or 0,
resources['lb_util_pct'] or 0, resources['pool_util_pct'], saturated), flush=True)
finally:
if not args.skip_restore:
print('restoring default 2 × 1.0 CPU topology...', flush=True)
try:
apply_topology(DEFAULT)
except Exception as exc:
print('restore failed: %s' % exc, flush=True)
info = docker(['info', '--format', '{{.NCPU}} {{.MemTotal}}']).split()
report = {
'hypothesis': (
'With a fixed total API CPU/RAM budget (2.0 CPU / 512M), two replicas outperform both one '
'(a single uvicorn worker cannot spend 2.0 CPU) and four (each worker is capped at 0.5 CPU, '
'p95 collapses). Growing per-replica CPU still raises RPS until that slice is no longer tiny. '
'PostgreSQL and HAProxy stay below their limits; extra replicas past 2 only add pool connections '
'to the same 1.0-CPU database. The confirmed improvement is 2×1.0 instead of 4×0.5 at the same budget.'
),
'host': {'ncpu': int(info[0]), 'memory_bytes': int(info[1])},
'profile': {
'concurrency': args.concurrency,
'warmup_seconds': args.warmup,
'measure_seconds': args.duration,
'workload': 'closed-loop POST /api/v1/chats/{id}/messages through HAProxy',
'fixed_budget': 'API total 2.0 CPU / 512M; postgres 1.0 CPU / 512M; pool 10 per replica',
},
'runs': runs,
'finding': finding(runs) if {r['id'] for r in runs} >= {c['id'] for c in CONFIGS} else 'partial run',
}
out_json = ROOT / 'evidence' / 'scale-cost.json'
out_txt = ROOT / 'evidence' / 'scale-cost.txt'
out_svg = ROOT / 'evidence' / 'scale-cost.svg'
out_json.write_text(json.dumps(report, ensure_ascii=False, indent=2) + '\n', encoding='utf-8')
write_txt(out_txt, report)
if {r['id'] for r in runs} >= {c['id'] for c in CONFIGS}:
write_svg(out_svg, runs)
print(report['finding'])
print('wrote %s, %s, %s' % (out_json, out_txt, out_svg if out_svg.exists() else '(no svg)'))
if __name__ == '__main__':
run(main)