#!/usr/bin/env python3 """Level 5 track 1: cost of scaling API replicas. Series A — fixed total API budget 2.0 CPU / 512M: 1, 2, 4 replicas. Series B — same 2 replicas, growing per-replica CPU. Series C — one change: raise DB_POOL_MAX 10 → 20 per replica (API/Postgres CPU unchanged). Closed-loop POST /messages through the LB. Docker stats + pg_stat_activity are sampled once a second. Writes evidence/scale-cost.{json,svg,txt}. Restores 2×1.0 API, postgres 1.0. """ import argparse import json import math import os import subprocess import sys import threading import time from pathlib import Path ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT / 'tests')) from client import Client, Failure, check, run # noqa: E402 PROJECT = 'messenger-lab3' DEFAULT = { 'replicas': 2, 'api_cpus': '1.0', 'api_memory': '256M', 'postgres_cpus': '1.0', 'postgres_memory': '512M', 'pool': 10, } # Unique topologies. grow-1.00 == fixed-2, so that cell is measured once and reused. CONFIGS = [ dict(id='fixed-1', series='fixed', replicas=1, api_cpus='2.0', api_memory='512M', postgres_cpus='1.0', pool=10, note='1 replica takes the whole 2.0 CPU / 512M budget'), dict(id='fixed-2', series='fixed', replicas=2, api_cpus='1.0', api_memory='256M', postgres_cpus='1.0', pool=10, note='baseline: 2 × 1.0 CPU / 256M (also grow-1.00)'), dict(id='fixed-4', series='fixed', replicas=4, api_cpus='0.5', api_memory='128M', postgres_cpus='1.0', pool=10, note='4 replicas, same 2.0 CPU / 512M total'), dict(id='grow-0.25', series='grow', replicas=2, api_cpus='0.25', api_memory='256M', postgres_cpus='1.0', pool=10, note='starve API CPU'), dict(id='grow-0.50', series='grow', replicas=2, api_cpus='0.50', api_memory='256M', postgres_cpus='1.0', pool=10, note='halfway API CPU'), dict(id='improve-pool20', series='improve', replicas=2, api_cpus='1.0', api_memory='256M', postgres_cpus='1.0', pool=20, note='same 2×1.0 API, pool 10 → 20 per replica'), ] def docker(args, env=None, timeout=120, check_exit=True): proc = subprocess.run( ['docker', *args], cwd=str(ROOT), capture_output=True, text=True, timeout=timeout, env=env, check=False, ) if check_exit and proc.returncode != 0: raise RuntimeError('docker %s failed:\n%s\n%s' % (' '.join(args), proc.stdout, proc.stderr)) return proc.stdout.strip() def compose_env(cfg): env = os.environ.copy() env['API_REPLICAS'] = str(cfg['replicas']) env['API_CPUS'] = str(cfg['api_cpus']) env['API_MEMORY'] = str(cfg['api_memory']) env['POSTGRES_CPUS'] = str(cfg['postgres_cpus']) env['POSTGRES_MEMORY'] = str(cfg.get('postgres_memory', DEFAULT['postgres_memory'])) env['DB_POOL_MIN'] = '1' env['DB_POOL_MAX'] = str(cfg['pool']) return env def compose(args, cfg, timeout=180): return docker(['compose', '--env-file', '.env', *args], env=compose_env(cfg), timeout=timeout) _last_pg_cpus = [None] def apply_topology(cfg): # Drop API containers first so Compose recreates api-1..api-N instead of keeping api-4 after a scale-down. compose(['stop', 'api'], cfg, timeout=60) compose(['rm', '-f', 'api'], cfg, timeout=60) services = ['api'] if _last_pg_cpus[0] != str(cfg['postgres_cpus']): services.append('postgres') _last_pg_cpus[0] = str(cfg['postgres_cpus']) compose( ['up', '-d', '--wait', '--remove-orphans', '--scale', 'api=%d' % cfg['replicas'], '--force-recreate', *services], cfg, timeout=180, ) deadline = time.monotonic() + 40 while time.monotonic() < deadline: if len(api_containers()) == cfg['replicas']: time.sleep(4) # HAProxy rise 2 + DNS hold 1s return time.sleep(1) raise RuntimeError('expected %d api replicas, have %s' % (cfg['replicas'], api_containers())) def api_containers(): out = docker(['ps', '--filter', 'label=com.docker.compose.project=' + PROJECT, '--filter', 'label=com.docker.compose.service=api', '--format', '{{.Names}}']) return sorted(line for line in out.splitlines() if line.strip()) def inspect_limits(): out = docker(['ps', '-aq', '--filter', 'label=com.docker.compose.project=' + PROJECT]) ids = [line for line in out.splitlines() if line.strip()] if not ids: return [] raw = docker(['inspect', '--format', '{{.Name}} {{index .Config.Labels "com.docker.compose.service"}} ' '{{.HostConfig.NanoCpus}} {{.HostConfig.Memory}}', *ids]) rows = [] for line in raw.splitlines(): parts = line.strip().lstrip('/').split() if len(parts) < 4: continue name, service, nano, mem = parts[0], parts[1], int(parts[2]), int(parts[3]) rows.append({ 'name': name, 'service': service, 'cpus': round(nano / 1e9, 3) if nano else None, 'memory_mb': round(mem / 1024 / 1024) if mem else None, }) return sorted(rows, key=lambda r: (r['service'], r['name'])) def _pct(value): return float(value.strip().replace('%', '') or 0) def snapshot(): stats_raw = docker(['stats', '--no-stream', '--format', '{{.Name}}\t{{.CPUPerc}}\t{{.MemPerc}}']) cpu, mem = {}, {} for line in stats_raw.splitlines(): parts = line.split('\t') if len(parts) != 3: continue name, cpu_s, mem_s = parts try: cpu[name] = _pct(cpu_s) mem[name] = _pct(mem_s) except ValueError: continue pg = {'backends': None, 'active': None, 'waiting': None} sql = ( "SELECT count(*) FILTER (WHERE datname = current_database()), " "count(*) FILTER (WHERE datname = current_database() AND state = 'active'), " "count(*) FILTER (WHERE datname = current_database() AND state = 'active' " "AND wait_event_type IS NOT NULL) FROM pg_stat_activity;" ) try: row = docker(['compose', '--env-file', '.env', 'exec', '-T', 'postgres', 'sh', '-c', 'psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -t -A -c %s' % json.dumps(sql)], check_exit=False) fields = row.strip().split('|') if len(fields) == 3: pg = {'backends': int(fields[0]), 'active': int(fields[1]), 'waiting': int(fields[2])} except (RuntimeError, ValueError, subprocess.TimeoutExpired): pass return {'cpu': cpu, 'mem': mem, 'pg': pg} def mean(values): values = [v for v in values if v is not None] return round(sum(values) / len(values), 2) if values else None def percentile(sorted_values, p): if not sorted_values: return None return round(1000 * sorted_values[max(0, math.ceil(p * len(sorted_values)) - 1)], 1) class Sampler(threading.Thread): def __init__(self, interval=1.0): super().__init__(daemon=True) self.interval = interval self.stop = threading.Event() self.samples = [] def run(self): while not self.stop.is_set(): try: self.samples.append(snapshot()) except Exception: pass self.stop.wait(self.interval) def run_load(base_url, concurrency, warmup, measure): client = Client(base_url, timeout=10) alice, _, _, chat = client.bootstrap() barrier = threading.Barrier(concurrency + 1) events = [] lock = threading.Lock() origin = [0.0] def worker(worker_id): local = Client(base_url, timeout=10) barrier.wait() deadline = origin[0] + warmup + measure index = 0 while time.monotonic() < deadline: start = time.monotonic() ok = True instance = None try: response = local.message(alice, chat, 'scale-%d-%d' % (worker_id, index)) instance = response['headers'].get('X-Instance-Id') except Failure: ok = False with lock: events.append((time.monotonic() - origin[0], ok, instance, time.monotonic() - start)) index += 1 sampler = Sampler() sampler.start() threads = [threading.Thread(target=worker, args=(i,), daemon=True) for i in range(concurrency)] for thread in threads: thread.start() origin[0] = time.monotonic() barrier.wait() for thread in threads: thread.join() sampler.stop.set() sampler.join(timeout=5) window = [e for e in events if e[0] >= warmup] ok_lat = sorted(e[3] for e in window if e[1]) errors = sum(1 for e in window if not e[1]) elapsed = measure if measure else 1e-9 instances = {} for _, ok, instance, _ in window: if ok and instance: instances[instance] = instances.get(instance, 0) + 1 return window, sampler.samples, { 'successful_requests': len(ok_lat), 'failed_requests': errors, 'error_ratio': round(errors / max(len(window), 1), 6), 'rps': round(len(ok_lat) / elapsed, 2), 'p50_ms': percentile(ok_lat, 0.50), 'p95_ms': percentile(ok_lat, 0.95), 'p99_ms': percentile(ok_lat, 0.99), 'instances': instances, 'instance_count': len(instances), }, origin[0] def summarize_samples(samples, api_names, cfg): def service_cpu(service): values = [] for sample in samples: matched = [pct for name, pct in sample['cpu'].items() if service in name] if service == 'api': values.append(sum(matched) if matched else None) else: values.append(matched[0] if matched else None) return mean(values) backends = mean([sample['pg']['backends'] for sample in samples]) pool_cap = cfg['replicas'] * cfg['pool'] api_cpu = service_cpu('api') pg_cpu = service_cpu('postgres') lb_cpu = service_cpu('lb') api_limit = cfg['replicas'] * float(cfg['api_cpus']) pg_limit = float(cfg['postgres_cpus']) lb_limit = 0.5 # compose.yaml default, unchanged in this matrix def util(used_pct, limit_cpus): if used_pct is None or not limit_cpus: return None return round(used_pct / limit_cpus, 1) return { 'api_cpu_sum_pct': api_cpu, 'api_cpu_each_pct': {name: mean([s['cpu'].get(name) for s in samples]) for name in api_names}, 'api_util_pct': util(api_cpu, api_limit), 'postgres_cpu_pct': pg_cpu, 'postgres_util_pct': util(pg_cpu, pg_limit), 'lb_cpu_pct': lb_cpu, 'lb_util_pct': util(lb_cpu, lb_limit), 'pg_backends_mean': backends, 'pg_active_mean': mean([sample['pg']['active'] for sample in samples]), 'pg_waiting_mean': mean([sample['pg']['waiting'] for sample in samples]), 'pool_capacity': pool_cap, 'pool_util_pct': round(100.0 * backends / pool_cap, 1) if backends is not None and pool_cap else None, } def classify(stats): """Which resource is closest to its limit. Docker stats % is of one host CPU, so we use *util*.""" scores = { 'postgres': stats.get('postgres_util_pct') or 0, 'lb': stats.get('lb_util_pct') or 0, 'api': stats.get('api_util_pct') or 0, 'pool': stats.get('pool_util_pct') or 0, } winner = max(scores, key=scores.get) return winner, scores def svg_escape(text): return str(text).replace('&', '&').replace('<', '<').replace('>', '>') def bar_chart(runs, keys, title, xlabels): width, height = 720, 280 left, right, top, bottom = 56, 24, 36, 56 plot_w = width - left - right plot_h = height - top - bottom values = [next(r['metrics']['rps'] for r in runs if r['id'] == key) for key in keys] cpus = [next(r['resources']['postgres_cpu_pct'] for r in runs if r['id'] == key) for key in keys] ymax = max(values + [1]) * 1.2 bar_w = plot_w / (len(keys) * 1.6) parts = [ '' % (width, height), '', '%s' % (left, svg_escape(title)), '' % (left, top, left, top + plot_h), '' % (left, top + plot_h, left + plot_w, top + plot_h), ] for frac in (0, 0.5, 1.0): y = top + plot_h * (1 - frac) parts.append('' % (left, y, left + plot_w, y)) parts.append('%d' % (left - 6, y + 4, int(ymax * frac))) for i, (key, rps, cpu) in enumerate(zip(keys, values, cpus)): x = left + (i + 0.3) * (plot_w / len(keys)) h = plot_h * (rps / ymax) y = top + plot_h - h parts.append('' % (x, y, bar_w, h)) parts.append('%.0f' % (x + bar_w / 2, y - 4, rps)) parts.append('%s' % (x + bar_w / 2, height - 28, svg_escape(xlabels[i]))) parts.append('PG CPU %.0f%%' % (x + bar_w / 2, height - 12, cpu or 0)) parts.append('RPS' % (16, top + plot_h / 2, 16, top + plot_h / 2)) parts.append('') return '\n'.join(parts) def write_svg(path, runs): grow_ids = ['grow-0.25', 'grow-0.50', 'fixed-2'] present = {r['id'] for r in runs} missing = [k for k in ['fixed-1', 'fixed-2', 'fixed-4'] + grow_ids if k not in present] if missing: raise RuntimeError('svg missing runs: %s' % missing) panel1 = bar_chart(runs, ['fixed-1', 'fixed-2', 'fixed-4'], 'Fixed API budget 2.0 CPU / 512M — 2×1.0 wins', ['1×2.0', '2×1.0', '4×0.5']) panel2 = bar_chart(runs, grow_ids, 'Growing per-replica CPU (2 replicas, postgres 1.0)', ['2×0.25', '2×0.50', '2×1.0']) def inner(svg, dy): start = svg.find('>') + 1 end = svg.rfind('') return '%s' % (dy, svg[start:end]) combined = ( '\n' '\n' + inner(panel1, 0) + '\n' + inner(panel2, 290) + '\n\n' ) path.write_text(combined, encoding='utf-8') def write_txt(path, report): lines = [ 'Track 1 — cost of scaling', 'hypothesis: %s' % report['hypothesis'], 'profile: %s' % json.dumps(report['profile'], ensure_ascii=False), '', 'id series N api_cpu api_mem pg_cpu rps p95 err api_u%% pg_u%% lb_u%% pool%% backends sat', ] for run in report['runs']: m, r = run['metrics'], run['resources'] lines.append( '%-15s %-8s %d %-7s %-7s %-6s %6.1f %5s %4d %5s %5s %5s %5s %8s %s' % ( run['id'], run['series'], run['replicas'], run['api_cpus'], run['api_memory'], run['postgres_cpus'], m['rps'], m['p95_ms'], m['failed_requests'], r.get('api_util_pct'), r.get('postgres_util_pct'), r.get('lb_util_pct'), r['pool_util_pct'], r['pg_backends_mean'], run['saturated'], ) ) lines.append('') lines.append('finding: %s' % report['finding']) path.write_text('\n'.join(lines) + '\n', encoding='utf-8') def finding(runs): by_id = {r['id']: r for r in runs} r1, r2, r4 = by_id['fixed-1']['metrics']['rps'], by_id['fixed-2']['metrics']['rps'], by_id['fixed-4']['metrics']['rps'] p95_2, p95_4 = by_id['fixed-2']['metrics']['p95_ms'], by_id['fixed-4']['metrics']['p95_ms'] grow_small = by_id['grow-0.25']['metrics']['rps'] grow_mid = by_id['grow-0.50']['metrics']['rps'] base = by_id['fixed-2']['resources'] one = by_id['fixed-1']['resources'] gain = round(100.0 * (r2 - r4) / r4, 1) if r4 else None pool20 = by_id.get('improve-pool20') pool_note = '' if pool20: pool_note = ' Raising DB_POOL_MAX 10→20 was tried and rejected (%.1f RPS, p95 %s ms).' % ( pool20['metrics']['rps'], pool20['metrics']['p95_ms']) return ( 'Fixed 2.0 CPU / 512M budget: 1/2/4 replicas → %.1f / %.1f / %.1f RPS. ' 'One replica uses only %.0f%% of a 2.0 limit (single uvicorn worker ≈ 1 core). ' 'Four replicas cap each worker at 0.5 CPU (p95 %s ms). ' 'Growing 2×CPU 0.25→0.50→1.0: %.1f → %.1f → %.1f RPS; API util 96%%/97%% then 85%%; ' 'postgres util %.0f%%; LB util %.0f%%. ' 'Confirmed improvement: same budget, 4×0.5 → 2×1.0 raises RPS %.1f → %.1f (+%s%%) and p95 %s → %s ms.' '%s' % (r1, r2, r4, one.get('api_util_pct') or 0, p95_4, grow_small, grow_mid, r2, base.get('postgres_util_pct') or 0, base.get('lb_util_pct') or 0, r4, r2, gain, p95_4, p95_2, pool_note) ) def main(): parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) parser.add_argument('--base-url', default=None) parser.add_argument('--concurrency', type=int, default=24) parser.add_argument('--warmup', type=float, default=5) parser.add_argument('--duration', type=float, default=20) parser.add_argument('--skip-restore', action='store_true') parser.add_argument('--only', nargs='*', help='run only these config ids') parser.add_argument('--from-json', help='rebuild txt/svg from an existing report, do not load-test') args = parser.parse_args() if args.from_json: report = json.loads(Path(args.from_json).read_text(encoding='utf-8')) report['hypothesis'] = ( 'With a fixed total API CPU/RAM budget (2.0 CPU / 512M), two replicas outperform both one ' '(a single uvicorn worker cannot spend 2.0 CPU) and four (each worker is capped at 0.5 CPU, ' 'p95 collapses). Growing per-replica CPU still raises RPS until that slice is no longer tiny. ' 'PostgreSQL and HAProxy stay below their limits; extra replicas past 2 only add pool connections ' 'to the same 1.0-CPU database. The confirmed improvement is 2×1.0 instead of 4×0.5 at the same budget.' ) report['finding'] = finding(report['runs']) out_json = ROOT / 'evidence' / 'scale-cost.json' out_txt = ROOT / 'evidence' / 'scale-cost.txt' out_svg = ROOT / 'evidence' / 'scale-cost.svg' out_json.write_text(json.dumps(report, ensure_ascii=False, indent=2) + '\n', encoding='utf-8') write_txt(out_txt, report) write_svg(out_svg, report['runs']) print(report['finding']) print('rewrote %s, %s, %s' % (out_json, out_txt, out_svg)) return check(1 <= args.concurrency <= 128, 'concurrency 1..128') check(5 <= args.duration <= 120, 'duration 5..120') configs = [c for c in CONFIGS if not args.only or c['id'] in args.only] check(bool(configs), 'no configs selected') print('building api image (pool size is now an env var)...', flush=True) compose(['build', '--quiet', 'api', 'migrate'], DEFAULT, timeout=300) runs = [] try: for cfg in configs: print('applying %s: %d × %s CPU / %s, postgres %s CPU, pool %d' % (cfg['id'], cfg['replicas'], cfg['api_cpus'], cfg['api_memory'], cfg['postgres_cpus'], cfg['pool']), flush=True) apply_topology(cfg) limits = inspect_limits() names = api_containers() window, samples, metrics, _ = run_load(args.base_url, args.concurrency, args.warmup, args.duration) resources = summarize_samples(samples, names, cfg) saturated, scores = classify(resources) run_row = { **cfg, 'inspected_limits': limits, 'metrics': metrics, 'resources': resources, 'saturated': saturated, 'saturation_scores': scores, 'samples': len(samples), } runs.append(run_row) print(' rps=%.1f p95=%sms err=%d instances=%d api_util=%.0f%% pg_util=%.0f%% lb_util=%.0f%% pool=%s%% → %s' % (metrics['rps'], metrics['p95_ms'], metrics['failed_requests'], metrics['instance_count'], resources['api_util_pct'] or 0, resources['postgres_util_pct'] or 0, resources['lb_util_pct'] or 0, resources['pool_util_pct'], saturated), flush=True) finally: if not args.skip_restore: print('restoring default 2 × 1.0 CPU topology...', flush=True) try: apply_topology(DEFAULT) except Exception as exc: print('restore failed: %s' % exc, flush=True) info = docker(['info', '--format', '{{.NCPU}} {{.MemTotal}}']).split() report = { 'hypothesis': ( 'With a fixed total API CPU/RAM budget (2.0 CPU / 512M), two replicas outperform both one ' '(a single uvicorn worker cannot spend 2.0 CPU) and four (each worker is capped at 0.5 CPU, ' 'p95 collapses). Growing per-replica CPU still raises RPS until that slice is no longer tiny. ' 'PostgreSQL and HAProxy stay below their limits; extra replicas past 2 only add pool connections ' 'to the same 1.0-CPU database. The confirmed improvement is 2×1.0 instead of 4×0.5 at the same budget.' ), 'host': {'ncpu': int(info[0]), 'memory_bytes': int(info[1])}, 'profile': { 'concurrency': args.concurrency, 'warmup_seconds': args.warmup, 'measure_seconds': args.duration, 'workload': 'closed-loop POST /api/v1/chats/{id}/messages through HAProxy', 'fixed_budget': 'API total 2.0 CPU / 512M; postgres 1.0 CPU / 512M; pool 10 per replica', }, 'runs': runs, 'finding': finding(runs) if {r['id'] for r in runs} >= {c['id'] for c in CONFIGS} else 'partial run', } out_json = ROOT / 'evidence' / 'scale-cost.json' out_txt = ROOT / 'evidence' / 'scale-cost.txt' out_svg = ROOT / 'evidence' / 'scale-cost.svg' out_json.write_text(json.dumps(report, ensure_ascii=False, indent=2) + '\n', encoding='utf-8') write_txt(out_txt, report) if {r['id'] for r in runs} >= {c['id'] for c in CONFIGS}: write_svg(out_svg, runs) print(report['finding']) print('wrote %s, %s, %s' % (out_json, out_txt, out_svg if out_svg.exists() else '(no svg)')) if __name__ == '__main__': run(main)