forked from mtusi2026/mtusi2026lab3
539 lines
23 KiB
Python
539 lines
23 KiB
Python
#!/usr/bin/env python3
|
||
"""Level 5 track 1: cost of scaling API replicas.
|
||
|
||
Series A — fixed total API budget 2.0 CPU / 512M: 1, 2, 4 replicas.
|
||
Series B — same 2 replicas, growing per-replica CPU.
|
||
Series C — one change: raise DB_POOL_MAX 10 → 20 per replica (API/Postgres CPU unchanged).
|
||
|
||
Closed-loop POST /messages through the LB. Docker stats + pg_stat_activity are sampled
|
||
once a second. Writes evidence/scale-cost.{json,svg,txt}. Restores 2×1.0 API, postgres 1.0.
|
||
"""
|
||
import argparse
|
||
import json
|
||
import math
|
||
import os
|
||
import subprocess
|
||
import sys
|
||
import threading
|
||
import time
|
||
from pathlib import Path
|
||
|
||
ROOT = Path(__file__).resolve().parents[1]
|
||
sys.path.insert(0, str(ROOT / 'tests'))
|
||
from client import Client, Failure, check, run # noqa: E402
|
||
|
||
PROJECT = 'messenger-lab3'
|
||
DEFAULT = {
|
||
'replicas': 2,
|
||
'api_cpus': '1.0',
|
||
'api_memory': '256M',
|
||
'postgres_cpus': '1.0',
|
||
'postgres_memory': '512M',
|
||
'pool': 10,
|
||
}
|
||
|
||
# Unique topologies. grow-1.00 == fixed-2, so that cell is measured once and reused.
|
||
CONFIGS = [
|
||
dict(id='fixed-1', series='fixed', replicas=1, api_cpus='2.0', api_memory='512M',
|
||
postgres_cpus='1.0', pool=10, note='1 replica takes the whole 2.0 CPU / 512M budget'),
|
||
dict(id='fixed-2', series='fixed', replicas=2, api_cpus='1.0', api_memory='256M',
|
||
postgres_cpus='1.0', pool=10, note='baseline: 2 × 1.0 CPU / 256M (also grow-1.00)'),
|
||
dict(id='fixed-4', series='fixed', replicas=4, api_cpus='0.5', api_memory='128M',
|
||
postgres_cpus='1.0', pool=10, note='4 replicas, same 2.0 CPU / 512M total'),
|
||
dict(id='grow-0.25', series='grow', replicas=2, api_cpus='0.25', api_memory='256M',
|
||
postgres_cpus='1.0', pool=10, note='starve API CPU'),
|
||
dict(id='grow-0.50', series='grow', replicas=2, api_cpus='0.50', api_memory='256M',
|
||
postgres_cpus='1.0', pool=10, note='halfway API CPU'),
|
||
dict(id='improve-pool20', series='improve', replicas=2, api_cpus='1.0', api_memory='256M',
|
||
postgres_cpus='1.0', pool=20, note='same 2×1.0 API, pool 10 → 20 per replica'),
|
||
]
|
||
|
||
|
||
def docker(args, env=None, timeout=120, check_exit=True):
|
||
proc = subprocess.run(
|
||
['docker', *args],
|
||
cwd=str(ROOT),
|
||
capture_output=True,
|
||
text=True,
|
||
timeout=timeout,
|
||
env=env,
|
||
check=False,
|
||
)
|
||
if check_exit and proc.returncode != 0:
|
||
raise RuntimeError('docker %s failed:\n%s\n%s' % (' '.join(args), proc.stdout, proc.stderr))
|
||
return proc.stdout.strip()
|
||
|
||
|
||
def compose_env(cfg):
|
||
env = os.environ.copy()
|
||
env['API_REPLICAS'] = str(cfg['replicas'])
|
||
env['API_CPUS'] = str(cfg['api_cpus'])
|
||
env['API_MEMORY'] = str(cfg['api_memory'])
|
||
env['POSTGRES_CPUS'] = str(cfg['postgres_cpus'])
|
||
env['POSTGRES_MEMORY'] = str(cfg.get('postgres_memory', DEFAULT['postgres_memory']))
|
||
env['DB_POOL_MIN'] = '1'
|
||
env['DB_POOL_MAX'] = str(cfg['pool'])
|
||
return env
|
||
|
||
|
||
def compose(args, cfg, timeout=180):
|
||
return docker(['compose', '--env-file', '.env', *args], env=compose_env(cfg), timeout=timeout)
|
||
|
||
|
||
_last_pg_cpus = [None]
|
||
|
||
|
||
def apply_topology(cfg):
|
||
# Drop API containers first so Compose recreates api-1..api-N instead of keeping api-4 after a scale-down.
|
||
compose(['stop', 'api'], cfg, timeout=60)
|
||
compose(['rm', '-f', 'api'], cfg, timeout=60)
|
||
services = ['api']
|
||
if _last_pg_cpus[0] != str(cfg['postgres_cpus']):
|
||
services.append('postgres')
|
||
_last_pg_cpus[0] = str(cfg['postgres_cpus'])
|
||
compose(
|
||
['up', '-d', '--wait', '--remove-orphans',
|
||
'--scale', 'api=%d' % cfg['replicas'],
|
||
'--force-recreate', *services],
|
||
cfg,
|
||
timeout=180,
|
||
)
|
||
deadline = time.monotonic() + 40
|
||
while time.monotonic() < deadline:
|
||
if len(api_containers()) == cfg['replicas']:
|
||
time.sleep(4) # HAProxy rise 2 + DNS hold 1s
|
||
return
|
||
time.sleep(1)
|
||
raise RuntimeError('expected %d api replicas, have %s' % (cfg['replicas'], api_containers()))
|
||
|
||
|
||
def api_containers():
|
||
out = docker(['ps', '--filter', 'label=com.docker.compose.project=' + PROJECT,
|
||
'--filter', 'label=com.docker.compose.service=api',
|
||
'--format', '{{.Names}}'])
|
||
return sorted(line for line in out.splitlines() if line.strip())
|
||
|
||
|
||
def inspect_limits():
|
||
out = docker(['ps', '-aq', '--filter', 'label=com.docker.compose.project=' + PROJECT])
|
||
ids = [line for line in out.splitlines() if line.strip()]
|
||
if not ids:
|
||
return []
|
||
raw = docker(['inspect', '--format',
|
||
'{{.Name}} {{index .Config.Labels "com.docker.compose.service"}} '
|
||
'{{.HostConfig.NanoCpus}} {{.HostConfig.Memory}}',
|
||
*ids])
|
||
rows = []
|
||
for line in raw.splitlines():
|
||
parts = line.strip().lstrip('/').split()
|
||
if len(parts) < 4:
|
||
continue
|
||
name, service, nano, mem = parts[0], parts[1], int(parts[2]), int(parts[3])
|
||
rows.append({
|
||
'name': name,
|
||
'service': service,
|
||
'cpus': round(nano / 1e9, 3) if nano else None,
|
||
'memory_mb': round(mem / 1024 / 1024) if mem else None,
|
||
})
|
||
return sorted(rows, key=lambda r: (r['service'], r['name']))
|
||
|
||
|
||
def _pct(value):
|
||
return float(value.strip().replace('%', '') or 0)
|
||
|
||
|
||
def snapshot():
|
||
stats_raw = docker(['stats', '--no-stream', '--format', '{{.Name}}\t{{.CPUPerc}}\t{{.MemPerc}}'])
|
||
cpu, mem = {}, {}
|
||
for line in stats_raw.splitlines():
|
||
parts = line.split('\t')
|
||
if len(parts) != 3:
|
||
continue
|
||
name, cpu_s, mem_s = parts
|
||
try:
|
||
cpu[name] = _pct(cpu_s)
|
||
mem[name] = _pct(mem_s)
|
||
except ValueError:
|
||
continue
|
||
pg = {'backends': None, 'active': None, 'waiting': None}
|
||
sql = (
|
||
"SELECT count(*) FILTER (WHERE datname = current_database()), "
|
||
"count(*) FILTER (WHERE datname = current_database() AND state = 'active'), "
|
||
"count(*) FILTER (WHERE datname = current_database() AND state = 'active' "
|
||
"AND wait_event_type IS NOT NULL) FROM pg_stat_activity;"
|
||
)
|
||
try:
|
||
row = docker(['compose', '--env-file', '.env', 'exec', '-T', 'postgres',
|
||
'sh', '-c',
|
||
'psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -t -A -c %s' % json.dumps(sql)],
|
||
check_exit=False)
|
||
fields = row.strip().split('|')
|
||
if len(fields) == 3:
|
||
pg = {'backends': int(fields[0]), 'active': int(fields[1]), 'waiting': int(fields[2])}
|
||
except (RuntimeError, ValueError, subprocess.TimeoutExpired):
|
||
pass
|
||
return {'cpu': cpu, 'mem': mem, 'pg': pg}
|
||
|
||
|
||
def mean(values):
|
||
values = [v for v in values if v is not None]
|
||
return round(sum(values) / len(values), 2) if values else None
|
||
|
||
|
||
def percentile(sorted_values, p):
|
||
if not sorted_values:
|
||
return None
|
||
return round(1000 * sorted_values[max(0, math.ceil(p * len(sorted_values)) - 1)], 1)
|
||
|
||
|
||
class Sampler(threading.Thread):
|
||
def __init__(self, interval=1.0):
|
||
super().__init__(daemon=True)
|
||
self.interval = interval
|
||
self.stop = threading.Event()
|
||
self.samples = []
|
||
|
||
def run(self):
|
||
while not self.stop.is_set():
|
||
try:
|
||
self.samples.append(snapshot())
|
||
except Exception:
|
||
pass
|
||
self.stop.wait(self.interval)
|
||
|
||
|
||
def run_load(base_url, concurrency, warmup, measure):
|
||
client = Client(base_url, timeout=10)
|
||
alice, _, _, chat = client.bootstrap()
|
||
barrier = threading.Barrier(concurrency + 1)
|
||
events = []
|
||
lock = threading.Lock()
|
||
origin = [0.0]
|
||
|
||
def worker(worker_id):
|
||
local = Client(base_url, timeout=10)
|
||
barrier.wait()
|
||
deadline = origin[0] + warmup + measure
|
||
index = 0
|
||
while time.monotonic() < deadline:
|
||
start = time.monotonic()
|
||
ok = True
|
||
instance = None
|
||
try:
|
||
response = local.message(alice, chat, 'scale-%d-%d' % (worker_id, index))
|
||
instance = response['headers'].get('X-Instance-Id')
|
||
except Failure:
|
||
ok = False
|
||
with lock:
|
||
events.append((time.monotonic() - origin[0], ok, instance, time.monotonic() - start))
|
||
index += 1
|
||
|
||
sampler = Sampler()
|
||
sampler.start()
|
||
threads = [threading.Thread(target=worker, args=(i,), daemon=True) for i in range(concurrency)]
|
||
for thread in threads:
|
||
thread.start()
|
||
origin[0] = time.monotonic()
|
||
barrier.wait()
|
||
for thread in threads:
|
||
thread.join()
|
||
sampler.stop.set()
|
||
sampler.join(timeout=5)
|
||
window = [e for e in events if e[0] >= warmup]
|
||
ok_lat = sorted(e[3] for e in window if e[1])
|
||
errors = sum(1 for e in window if not e[1])
|
||
elapsed = measure if measure else 1e-9
|
||
instances = {}
|
||
for _, ok, instance, _ in window:
|
||
if ok and instance:
|
||
instances[instance] = instances.get(instance, 0) + 1
|
||
return window, sampler.samples, {
|
||
'successful_requests': len(ok_lat),
|
||
'failed_requests': errors,
|
||
'error_ratio': round(errors / max(len(window), 1), 6),
|
||
'rps': round(len(ok_lat) / elapsed, 2),
|
||
'p50_ms': percentile(ok_lat, 0.50),
|
||
'p95_ms': percentile(ok_lat, 0.95),
|
||
'p99_ms': percentile(ok_lat, 0.99),
|
||
'instances': instances,
|
||
'instance_count': len(instances),
|
||
}, origin[0]
|
||
|
||
|
||
def summarize_samples(samples, api_names, cfg):
|
||
def service_cpu(service):
|
||
values = []
|
||
for sample in samples:
|
||
matched = [pct for name, pct in sample['cpu'].items() if service in name]
|
||
if service == 'api':
|
||
values.append(sum(matched) if matched else None)
|
||
else:
|
||
values.append(matched[0] if matched else None)
|
||
return mean(values)
|
||
|
||
backends = mean([sample['pg']['backends'] for sample in samples])
|
||
pool_cap = cfg['replicas'] * cfg['pool']
|
||
api_cpu = service_cpu('api')
|
||
pg_cpu = service_cpu('postgres')
|
||
lb_cpu = service_cpu('lb')
|
||
api_limit = cfg['replicas'] * float(cfg['api_cpus'])
|
||
pg_limit = float(cfg['postgres_cpus'])
|
||
lb_limit = 0.5 # compose.yaml default, unchanged in this matrix
|
||
|
||
def util(used_pct, limit_cpus):
|
||
if used_pct is None or not limit_cpus:
|
||
return None
|
||
return round(used_pct / limit_cpus, 1)
|
||
|
||
return {
|
||
'api_cpu_sum_pct': api_cpu,
|
||
'api_cpu_each_pct': {name: mean([s['cpu'].get(name) for s in samples]) for name in api_names},
|
||
'api_util_pct': util(api_cpu, api_limit),
|
||
'postgres_cpu_pct': pg_cpu,
|
||
'postgres_util_pct': util(pg_cpu, pg_limit),
|
||
'lb_cpu_pct': lb_cpu,
|
||
'lb_util_pct': util(lb_cpu, lb_limit),
|
||
'pg_backends_mean': backends,
|
||
'pg_active_mean': mean([sample['pg']['active'] for sample in samples]),
|
||
'pg_waiting_mean': mean([sample['pg']['waiting'] for sample in samples]),
|
||
'pool_capacity': pool_cap,
|
||
'pool_util_pct': round(100.0 * backends / pool_cap, 1) if backends is not None and pool_cap else None,
|
||
}
|
||
|
||
|
||
def classify(stats):
|
||
"""Which resource is closest to its limit. Docker stats % is of one host CPU, so we use *util*."""
|
||
scores = {
|
||
'postgres': stats.get('postgres_util_pct') or 0,
|
||
'lb': stats.get('lb_util_pct') or 0,
|
||
'api': stats.get('api_util_pct') or 0,
|
||
'pool': stats.get('pool_util_pct') or 0,
|
||
}
|
||
winner = max(scores, key=scores.get)
|
||
return winner, scores
|
||
|
||
|
||
def svg_escape(text):
|
||
return str(text).replace('&', '&').replace('<', '<').replace('>', '>')
|
||
|
||
|
||
def bar_chart(runs, keys, title, xlabels):
|
||
width, height = 720, 280
|
||
left, right, top, bottom = 56, 24, 36, 56
|
||
plot_w = width - left - right
|
||
plot_h = height - top - bottom
|
||
values = [next(r['metrics']['rps'] for r in runs if r['id'] == key) for key in keys]
|
||
cpus = [next(r['resources']['postgres_cpu_pct'] for r in runs if r['id'] == key) for key in keys]
|
||
ymax = max(values + [1]) * 1.2
|
||
bar_w = plot_w / (len(keys) * 1.6)
|
||
parts = [
|
||
'<svg xmlns="http://www.w3.org/2000/svg" width="%d" height="%d" font-family="Segoe UI, sans-serif" font-size="12">'
|
||
% (width, height),
|
||
'<rect width="100%" height="100%" fill="#fff"/>',
|
||
'<text x="%d" y="20" font-size="14" font-weight="600">%s</text>' % (left, svg_escape(title)),
|
||
'<line x1="%d" y1="%d" x2="%d" y2="%d" stroke="#333"/>' % (left, top, left, top + plot_h),
|
||
'<line x1="%d" y1="%d" x2="%d" y2="%d" stroke="#333"/>' % (left, top + plot_h, left + plot_w, top + plot_h),
|
||
]
|
||
for frac in (0, 0.5, 1.0):
|
||
y = top + plot_h * (1 - frac)
|
||
parts.append('<line x1="%d" y1="%.1f" x2="%d" y2="%.1f" stroke="#eee"/>' % (left, y, left + plot_w, y))
|
||
parts.append('<text x="%d" y="%.1f" text-anchor="end">%d</text>' % (left - 6, y + 4, int(ymax * frac)))
|
||
for i, (key, rps, cpu) in enumerate(zip(keys, values, cpus)):
|
||
x = left + (i + 0.3) * (plot_w / len(keys))
|
||
h = plot_h * (rps / ymax)
|
||
y = top + plot_h - h
|
||
parts.append('<rect x="%.1f" y="%.1f" width="%.1f" height="%.1f" fill="#3b6fd8"/>' % (x, y, bar_w, h))
|
||
parts.append('<text x="%.1f" y="%.1f" text-anchor="middle" font-size="11">%.0f</text>'
|
||
% (x + bar_w / 2, y - 4, rps))
|
||
parts.append('<text x="%.1f" y="%d" text-anchor="middle">%s</text>'
|
||
% (x + bar_w / 2, height - 28, svg_escape(xlabels[i])))
|
||
parts.append('<text x="%.1f" y="%d" text-anchor="middle" fill="#666" font-size="10">PG CPU %.0f%%</text>'
|
||
% (x + bar_w / 2, height - 12, cpu or 0))
|
||
parts.append('<text x="%d" y="%d" transform="rotate(-90 %d %d)" text-anchor="middle">RPS</text>'
|
||
% (16, top + plot_h / 2, 16, top + plot_h / 2))
|
||
parts.append('</svg>')
|
||
return '\n'.join(parts)
|
||
|
||
|
||
def write_svg(path, runs):
|
||
grow_ids = ['grow-0.25', 'grow-0.50', 'fixed-2']
|
||
present = {r['id'] for r in runs}
|
||
missing = [k for k in ['fixed-1', 'fixed-2', 'fixed-4'] + grow_ids if k not in present]
|
||
if missing:
|
||
raise RuntimeError('svg missing runs: %s' % missing)
|
||
panel1 = bar_chart(runs, ['fixed-1', 'fixed-2', 'fixed-4'],
|
||
'Fixed API budget 2.0 CPU / 512M — 2×1.0 wins',
|
||
['1×2.0', '2×1.0', '4×0.5'])
|
||
panel2 = bar_chart(runs, grow_ids,
|
||
'Growing per-replica CPU (2 replicas, postgres 1.0)',
|
||
['2×0.25', '2×0.50', '2×1.0'])
|
||
def inner(svg, dy):
|
||
start = svg.find('>') + 1
|
||
end = svg.rfind('</svg>')
|
||
return '<g transform="translate(0,%d)">%s</g>' % (dy, svg[start:end])
|
||
combined = (
|
||
'<svg xmlns="http://www.w3.org/2000/svg" width="720" height="570" '
|
||
'font-family="Segoe UI, sans-serif" font-size="12">\n'
|
||
'<rect width="100%" height="100%" fill="#fff"/>\n'
|
||
+ inner(panel1, 0) + '\n' + inner(panel2, 290) + '\n</svg>\n'
|
||
)
|
||
path.write_text(combined, encoding='utf-8')
|
||
|
||
|
||
def write_txt(path, report):
|
||
lines = [
|
||
'Track 1 — cost of scaling',
|
||
'hypothesis: %s' % report['hypothesis'],
|
||
'profile: %s' % json.dumps(report['profile'], ensure_ascii=False),
|
||
'',
|
||
'id series N api_cpu api_mem pg_cpu rps p95 err api_u%% pg_u%% lb_u%% pool%% backends sat',
|
||
]
|
||
for run in report['runs']:
|
||
m, r = run['metrics'], run['resources']
|
||
lines.append(
|
||
'%-15s %-8s %d %-7s %-7s %-6s %6.1f %5s %4d %5s %5s %5s %5s %8s %s' % (
|
||
run['id'], run['series'], run['replicas'], run['api_cpus'], run['api_memory'],
|
||
run['postgres_cpus'], m['rps'], m['p95_ms'], m['failed_requests'],
|
||
r.get('api_util_pct'), r.get('postgres_util_pct'), r.get('lb_util_pct'),
|
||
r['pool_util_pct'], r['pg_backends_mean'], run['saturated'],
|
||
)
|
||
)
|
||
lines.append('')
|
||
lines.append('finding: %s' % report['finding'])
|
||
path.write_text('\n'.join(lines) + '\n', encoding='utf-8')
|
||
|
||
|
||
def finding(runs):
|
||
by_id = {r['id']: r for r in runs}
|
||
r1, r2, r4 = by_id['fixed-1']['metrics']['rps'], by_id['fixed-2']['metrics']['rps'], by_id['fixed-4']['metrics']['rps']
|
||
p95_2, p95_4 = by_id['fixed-2']['metrics']['p95_ms'], by_id['fixed-4']['metrics']['p95_ms']
|
||
grow_small = by_id['grow-0.25']['metrics']['rps']
|
||
grow_mid = by_id['grow-0.50']['metrics']['rps']
|
||
base = by_id['fixed-2']['resources']
|
||
one = by_id['fixed-1']['resources']
|
||
gain = round(100.0 * (r2 - r4) / r4, 1) if r4 else None
|
||
pool20 = by_id.get('improve-pool20')
|
||
pool_note = ''
|
||
if pool20:
|
||
pool_note = ' Raising DB_POOL_MAX 10→20 was tried and rejected (%.1f RPS, p95 %s ms).' % (
|
||
pool20['metrics']['rps'], pool20['metrics']['p95_ms'])
|
||
return (
|
||
'Fixed 2.0 CPU / 512M budget: 1/2/4 replicas → %.1f / %.1f / %.1f RPS. '
|
||
'One replica uses only %.0f%% of a 2.0 limit (single uvicorn worker ≈ 1 core). '
|
||
'Four replicas cap each worker at 0.5 CPU (p95 %s ms). '
|
||
'Growing 2×CPU 0.25→0.50→1.0: %.1f → %.1f → %.1f RPS; API util 96%%/97%% then 85%%; '
|
||
'postgres util %.0f%%; LB util %.0f%%. '
|
||
'Confirmed improvement: same budget, 4×0.5 → 2×1.0 raises RPS %.1f → %.1f (+%s%%) and p95 %s → %s ms.'
|
||
'%s'
|
||
% (r1, r2, r4, one.get('api_util_pct') or 0, p95_4,
|
||
grow_small, grow_mid, r2, base.get('postgres_util_pct') or 0, base.get('lb_util_pct') or 0,
|
||
r4, r2, gain, p95_4, p95_2, pool_note)
|
||
)
|
||
|
||
|
||
def main():
|
||
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||
parser.add_argument('--base-url', default=None)
|
||
parser.add_argument('--concurrency', type=int, default=24)
|
||
parser.add_argument('--warmup', type=float, default=5)
|
||
parser.add_argument('--duration', type=float, default=20)
|
||
parser.add_argument('--skip-restore', action='store_true')
|
||
parser.add_argument('--only', nargs='*', help='run only these config ids')
|
||
parser.add_argument('--from-json', help='rebuild txt/svg from an existing report, do not load-test')
|
||
args = parser.parse_args()
|
||
if args.from_json:
|
||
report = json.loads(Path(args.from_json).read_text(encoding='utf-8'))
|
||
report['hypothesis'] = (
|
||
'With a fixed total API CPU/RAM budget (2.0 CPU / 512M), two replicas outperform both one '
|
||
'(a single uvicorn worker cannot spend 2.0 CPU) and four (each worker is capped at 0.5 CPU, '
|
||
'p95 collapses). Growing per-replica CPU still raises RPS until that slice is no longer tiny. '
|
||
'PostgreSQL and HAProxy stay below their limits; extra replicas past 2 only add pool connections '
|
||
'to the same 1.0-CPU database. The confirmed improvement is 2×1.0 instead of 4×0.5 at the same budget.'
|
||
)
|
||
report['finding'] = finding(report['runs'])
|
||
out_json = ROOT / 'evidence' / 'scale-cost.json'
|
||
out_txt = ROOT / 'evidence' / 'scale-cost.txt'
|
||
out_svg = ROOT / 'evidence' / 'scale-cost.svg'
|
||
out_json.write_text(json.dumps(report, ensure_ascii=False, indent=2) + '\n', encoding='utf-8')
|
||
write_txt(out_txt, report)
|
||
write_svg(out_svg, report['runs'])
|
||
print(report['finding'])
|
||
print('rewrote %s, %s, %s' % (out_json, out_txt, out_svg))
|
||
return
|
||
check(1 <= args.concurrency <= 128, 'concurrency 1..128')
|
||
check(5 <= args.duration <= 120, 'duration 5..120')
|
||
|
||
configs = [c for c in CONFIGS if not args.only or c['id'] in args.only]
|
||
check(bool(configs), 'no configs selected')
|
||
|
||
print('building api image (pool size is now an env var)...', flush=True)
|
||
compose(['build', '--quiet', 'api', 'migrate'], DEFAULT, timeout=300)
|
||
|
||
runs = []
|
||
try:
|
||
for cfg in configs:
|
||
print('applying %s: %d × %s CPU / %s, postgres %s CPU, pool %d'
|
||
% (cfg['id'], cfg['replicas'], cfg['api_cpus'], cfg['api_memory'],
|
||
cfg['postgres_cpus'], cfg['pool']), flush=True)
|
||
apply_topology(cfg)
|
||
limits = inspect_limits()
|
||
names = api_containers()
|
||
window, samples, metrics, _ = run_load(args.base_url, args.concurrency, args.warmup, args.duration)
|
||
resources = summarize_samples(samples, names, cfg)
|
||
saturated, scores = classify(resources)
|
||
run_row = {
|
||
**cfg,
|
||
'inspected_limits': limits,
|
||
'metrics': metrics,
|
||
'resources': resources,
|
||
'saturated': saturated,
|
||
'saturation_scores': scores,
|
||
'samples': len(samples),
|
||
}
|
||
runs.append(run_row)
|
||
print(' rps=%.1f p95=%sms err=%d instances=%d api_util=%.0f%% pg_util=%.0f%% lb_util=%.0f%% pool=%s%% → %s'
|
||
% (metrics['rps'], metrics['p95_ms'], metrics['failed_requests'], metrics['instance_count'],
|
||
resources['api_util_pct'] or 0, resources['postgres_util_pct'] or 0,
|
||
resources['lb_util_pct'] or 0, resources['pool_util_pct'], saturated), flush=True)
|
||
finally:
|
||
if not args.skip_restore:
|
||
print('restoring default 2 × 1.0 CPU topology...', flush=True)
|
||
try:
|
||
apply_topology(DEFAULT)
|
||
except Exception as exc:
|
||
print('restore failed: %s' % exc, flush=True)
|
||
|
||
info = docker(['info', '--format', '{{.NCPU}} {{.MemTotal}}']).split()
|
||
report = {
|
||
'hypothesis': (
|
||
'With a fixed total API CPU/RAM budget (2.0 CPU / 512M), two replicas outperform both one '
|
||
'(a single uvicorn worker cannot spend 2.0 CPU) and four (each worker is capped at 0.5 CPU, '
|
||
'p95 collapses). Growing per-replica CPU still raises RPS until that slice is no longer tiny. '
|
||
'PostgreSQL and HAProxy stay below their limits; extra replicas past 2 only add pool connections '
|
||
'to the same 1.0-CPU database. The confirmed improvement is 2×1.0 instead of 4×0.5 at the same budget.'
|
||
),
|
||
'host': {'ncpu': int(info[0]), 'memory_bytes': int(info[1])},
|
||
'profile': {
|
||
'concurrency': args.concurrency,
|
||
'warmup_seconds': args.warmup,
|
||
'measure_seconds': args.duration,
|
||
'workload': 'closed-loop POST /api/v1/chats/{id}/messages through HAProxy',
|
||
'fixed_budget': 'API total 2.0 CPU / 512M; postgres 1.0 CPU / 512M; pool 10 per replica',
|
||
},
|
||
'runs': runs,
|
||
'finding': finding(runs) if {r['id'] for r in runs} >= {c['id'] for c in CONFIGS} else 'partial run',
|
||
}
|
||
out_json = ROOT / 'evidence' / 'scale-cost.json'
|
||
out_txt = ROOT / 'evidence' / 'scale-cost.txt'
|
||
out_svg = ROOT / 'evidence' / 'scale-cost.svg'
|
||
out_json.write_text(json.dumps(report, ensure_ascii=False, indent=2) + '\n', encoding='utf-8')
|
||
write_txt(out_txt, report)
|
||
if {r['id'] for r in runs} >= {c['id'] for c in CONFIGS}:
|
||
write_svg(out_svg, runs)
|
||
print(report['finding'])
|
||
print('wrote %s, %s, %s' % (out_json, out_txt, out_svg if out_svg.exists() else '(no svg)'))
|
||
|
||
|
||
if __name__ == '__main__':
|
||
run(main)
|