Files
2026-11-08 13:13:34 +03:00

539 lines
23 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env python3
"""Level 5 track 1: cost of scaling API replicas.
Series A — fixed total API budget 2.0 CPU / 512M: 1, 2, 4 replicas.
Series B — same 2 replicas, growing per-replica CPU.
Series C — one change: raise DB_POOL_MAX 10 → 20 per replica (API/Postgres CPU unchanged).
Closed-loop POST /messages through the LB. Docker stats + pg_stat_activity are sampled
once a second. Writes evidence/scale-cost.{json,svg,txt}. Restores 2×1.0 API, postgres 1.0.
"""
import argparse
import json
import math
import os
import subprocess
import sys
import threading
import time
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / 'tests'))
from client import Client, Failure, check, run # noqa: E402
PROJECT = 'messenger-lab3'
DEFAULT = {
'replicas': 2,
'api_cpus': '1.0',
'api_memory': '256M',
'postgres_cpus': '1.0',
'postgres_memory': '512M',
'pool': 10,
}
# Unique topologies. grow-1.00 == fixed-2, so that cell is measured once and reused.
CONFIGS = [
dict(id='fixed-1', series='fixed', replicas=1, api_cpus='2.0', api_memory='512M',
postgres_cpus='1.0', pool=10, note='1 replica takes the whole 2.0 CPU / 512M budget'),
dict(id='fixed-2', series='fixed', replicas=2, api_cpus='1.0', api_memory='256M',
postgres_cpus='1.0', pool=10, note='baseline: 2 × 1.0 CPU / 256M (also grow-1.00)'),
dict(id='fixed-4', series='fixed', replicas=4, api_cpus='0.5', api_memory='128M',
postgres_cpus='1.0', pool=10, note='4 replicas, same 2.0 CPU / 512M total'),
dict(id='grow-0.25', series='grow', replicas=2, api_cpus='0.25', api_memory='256M',
postgres_cpus='1.0', pool=10, note='starve API CPU'),
dict(id='grow-0.50', series='grow', replicas=2, api_cpus='0.50', api_memory='256M',
postgres_cpus='1.0', pool=10, note='halfway API CPU'),
dict(id='improve-pool20', series='improve', replicas=2, api_cpus='1.0', api_memory='256M',
postgres_cpus='1.0', pool=20, note='same 2×1.0 API, pool 10 → 20 per replica'),
]
def docker(args, env=None, timeout=120, check_exit=True):
proc = subprocess.run(
['docker', *args],
cwd=str(ROOT),
capture_output=True,
text=True,
timeout=timeout,
env=env,
check=False,
)
if check_exit and proc.returncode != 0:
raise RuntimeError('docker %s failed:\n%s\n%s' % (' '.join(args), proc.stdout, proc.stderr))
return proc.stdout.strip()
def compose_env(cfg):
env = os.environ.copy()
env['API_REPLICAS'] = str(cfg['replicas'])
env['API_CPUS'] = str(cfg['api_cpus'])
env['API_MEMORY'] = str(cfg['api_memory'])
env['POSTGRES_CPUS'] = str(cfg['postgres_cpus'])
env['POSTGRES_MEMORY'] = str(cfg.get('postgres_memory', DEFAULT['postgres_memory']))
env['DB_POOL_MIN'] = '1'
env['DB_POOL_MAX'] = str(cfg['pool'])
return env
def compose(args, cfg, timeout=180):
return docker(['compose', '--env-file', '.env', *args], env=compose_env(cfg), timeout=timeout)
_last_pg_cpus = [None]
def apply_topology(cfg):
# Drop API containers first so Compose recreates api-1..api-N instead of keeping api-4 after a scale-down.
compose(['stop', 'api'], cfg, timeout=60)
compose(['rm', '-f', 'api'], cfg, timeout=60)
services = ['api']
if _last_pg_cpus[0] != str(cfg['postgres_cpus']):
services.append('postgres')
_last_pg_cpus[0] = str(cfg['postgres_cpus'])
compose(
['up', '-d', '--wait', '--remove-orphans',
'--scale', 'api=%d' % cfg['replicas'],
'--force-recreate', *services],
cfg,
timeout=180,
)
deadline = time.monotonic() + 40
while time.monotonic() < deadline:
if len(api_containers()) == cfg['replicas']:
time.sleep(4) # HAProxy rise 2 + DNS hold 1s
return
time.sleep(1)
raise RuntimeError('expected %d api replicas, have %s' % (cfg['replicas'], api_containers()))
def api_containers():
out = docker(['ps', '--filter', 'label=com.docker.compose.project=' + PROJECT,
'--filter', 'label=com.docker.compose.service=api',
'--format', '{{.Names}}'])
return sorted(line for line in out.splitlines() if line.strip())
def inspect_limits():
out = docker(['ps', '-aq', '--filter', 'label=com.docker.compose.project=' + PROJECT])
ids = [line for line in out.splitlines() if line.strip()]
if not ids:
return []
raw = docker(['inspect', '--format',
'{{.Name}} {{index .Config.Labels "com.docker.compose.service"}} '
'{{.HostConfig.NanoCpus}} {{.HostConfig.Memory}}',
*ids])
rows = []
for line in raw.splitlines():
parts = line.strip().lstrip('/').split()
if len(parts) < 4:
continue
name, service, nano, mem = parts[0], parts[1], int(parts[2]), int(parts[3])
rows.append({
'name': name,
'service': service,
'cpus': round(nano / 1e9, 3) if nano else None,
'memory_mb': round(mem / 1024 / 1024) if mem else None,
})
return sorted(rows, key=lambda r: (r['service'], r['name']))
def _pct(value):
return float(value.strip().replace('%', '') or 0)
def snapshot():
stats_raw = docker(['stats', '--no-stream', '--format', '{{.Name}}\t{{.CPUPerc}}\t{{.MemPerc}}'])
cpu, mem = {}, {}
for line in stats_raw.splitlines():
parts = line.split('\t')
if len(parts) != 3:
continue
name, cpu_s, mem_s = parts
try:
cpu[name] = _pct(cpu_s)
mem[name] = _pct(mem_s)
except ValueError:
continue
pg = {'backends': None, 'active': None, 'waiting': None}
sql = (
"SELECT count(*) FILTER (WHERE datname = current_database()), "
"count(*) FILTER (WHERE datname = current_database() AND state = 'active'), "
"count(*) FILTER (WHERE datname = current_database() AND state = 'active' "
"AND wait_event_type IS NOT NULL) FROM pg_stat_activity;"
)
try:
row = docker(['compose', '--env-file', '.env', 'exec', '-T', 'postgres',
'sh', '-c',
'psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -t -A -c %s' % json.dumps(sql)],
check_exit=False)
fields = row.strip().split('|')
if len(fields) == 3:
pg = {'backends': int(fields[0]), 'active': int(fields[1]), 'waiting': int(fields[2])}
except (RuntimeError, ValueError, subprocess.TimeoutExpired):
pass
return {'cpu': cpu, 'mem': mem, 'pg': pg}
def mean(values):
values = [v for v in values if v is not None]
return round(sum(values) / len(values), 2) if values else None
def percentile(sorted_values, p):
if not sorted_values:
return None
return round(1000 * sorted_values[max(0, math.ceil(p * len(sorted_values)) - 1)], 1)
class Sampler(threading.Thread):
def __init__(self, interval=1.0):
super().__init__(daemon=True)
self.interval = interval
self.stop = threading.Event()
self.samples = []
def run(self):
while not self.stop.is_set():
try:
self.samples.append(snapshot())
except Exception:
pass
self.stop.wait(self.interval)
def run_load(base_url, concurrency, warmup, measure):
client = Client(base_url, timeout=10)
alice, _, _, chat = client.bootstrap()
barrier = threading.Barrier(concurrency + 1)
events = []
lock = threading.Lock()
origin = [0.0]
def worker(worker_id):
local = Client(base_url, timeout=10)
barrier.wait()
deadline = origin[0] + warmup + measure
index = 0
while time.monotonic() < deadline:
start = time.monotonic()
ok = True
instance = None
try:
response = local.message(alice, chat, 'scale-%d-%d' % (worker_id, index))
instance = response['headers'].get('X-Instance-Id')
except Failure:
ok = False
with lock:
events.append((time.monotonic() - origin[0], ok, instance, time.monotonic() - start))
index += 1
sampler = Sampler()
sampler.start()
threads = [threading.Thread(target=worker, args=(i,), daemon=True) for i in range(concurrency)]
for thread in threads:
thread.start()
origin[0] = time.monotonic()
barrier.wait()
for thread in threads:
thread.join()
sampler.stop.set()
sampler.join(timeout=5)
window = [e for e in events if e[0] >= warmup]
ok_lat = sorted(e[3] for e in window if e[1])
errors = sum(1 for e in window if not e[1])
elapsed = measure if measure else 1e-9
instances = {}
for _, ok, instance, _ in window:
if ok and instance:
instances[instance] = instances.get(instance, 0) + 1
return window, sampler.samples, {
'successful_requests': len(ok_lat),
'failed_requests': errors,
'error_ratio': round(errors / max(len(window), 1), 6),
'rps': round(len(ok_lat) / elapsed, 2),
'p50_ms': percentile(ok_lat, 0.50),
'p95_ms': percentile(ok_lat, 0.95),
'p99_ms': percentile(ok_lat, 0.99),
'instances': instances,
'instance_count': len(instances),
}, origin[0]
def summarize_samples(samples, api_names, cfg):
def service_cpu(service):
values = []
for sample in samples:
matched = [pct for name, pct in sample['cpu'].items() if service in name]
if service == 'api':
values.append(sum(matched) if matched else None)
else:
values.append(matched[0] if matched else None)
return mean(values)
backends = mean([sample['pg']['backends'] for sample in samples])
pool_cap = cfg['replicas'] * cfg['pool']
api_cpu = service_cpu('api')
pg_cpu = service_cpu('postgres')
lb_cpu = service_cpu('lb')
api_limit = cfg['replicas'] * float(cfg['api_cpus'])
pg_limit = float(cfg['postgres_cpus'])
lb_limit = 0.5 # compose.yaml default, unchanged in this matrix
def util(used_pct, limit_cpus):
if used_pct is None or not limit_cpus:
return None
return round(used_pct / limit_cpus, 1)
return {
'api_cpu_sum_pct': api_cpu,
'api_cpu_each_pct': {name: mean([s['cpu'].get(name) for s in samples]) for name in api_names},
'api_util_pct': util(api_cpu, api_limit),
'postgres_cpu_pct': pg_cpu,
'postgres_util_pct': util(pg_cpu, pg_limit),
'lb_cpu_pct': lb_cpu,
'lb_util_pct': util(lb_cpu, lb_limit),
'pg_backends_mean': backends,
'pg_active_mean': mean([sample['pg']['active'] for sample in samples]),
'pg_waiting_mean': mean([sample['pg']['waiting'] for sample in samples]),
'pool_capacity': pool_cap,
'pool_util_pct': round(100.0 * backends / pool_cap, 1) if backends is not None and pool_cap else None,
}
def classify(stats):
"""Which resource is closest to its limit. Docker stats % is of one host CPU, so we use *util*."""
scores = {
'postgres': stats.get('postgres_util_pct') or 0,
'lb': stats.get('lb_util_pct') or 0,
'api': stats.get('api_util_pct') or 0,
'pool': stats.get('pool_util_pct') or 0,
}
winner = max(scores, key=scores.get)
return winner, scores
def svg_escape(text):
return str(text).replace('&', '&amp;').replace('<', '&lt;').replace('>', '&gt;')
def bar_chart(runs, keys, title, xlabels):
width, height = 720, 280
left, right, top, bottom = 56, 24, 36, 56
plot_w = width - left - right
plot_h = height - top - bottom
values = [next(r['metrics']['rps'] for r in runs if r['id'] == key) for key in keys]
cpus = [next(r['resources']['postgres_cpu_pct'] for r in runs if r['id'] == key) for key in keys]
ymax = max(values + [1]) * 1.2
bar_w = plot_w / (len(keys) * 1.6)
parts = [
'<svg xmlns="http://www.w3.org/2000/svg" width="%d" height="%d" font-family="Segoe UI, sans-serif" font-size="12">'
% (width, height),
'<rect width="100%" height="100%" fill="#fff"/>',
'<text x="%d" y="20" font-size="14" font-weight="600">%s</text>' % (left, svg_escape(title)),
'<line x1="%d" y1="%d" x2="%d" y2="%d" stroke="#333"/>' % (left, top, left, top + plot_h),
'<line x1="%d" y1="%d" x2="%d" y2="%d" stroke="#333"/>' % (left, top + plot_h, left + plot_w, top + plot_h),
]
for frac in (0, 0.5, 1.0):
y = top + plot_h * (1 - frac)
parts.append('<line x1="%d" y1="%.1f" x2="%d" y2="%.1f" stroke="#eee"/>' % (left, y, left + plot_w, y))
parts.append('<text x="%d" y="%.1f" text-anchor="end">%d</text>' % (left - 6, y + 4, int(ymax * frac)))
for i, (key, rps, cpu) in enumerate(zip(keys, values, cpus)):
x = left + (i + 0.3) * (plot_w / len(keys))
h = plot_h * (rps / ymax)
y = top + plot_h - h
parts.append('<rect x="%.1f" y="%.1f" width="%.1f" height="%.1f" fill="#3b6fd8"/>' % (x, y, bar_w, h))
parts.append('<text x="%.1f" y="%.1f" text-anchor="middle" font-size="11">%.0f</text>'
% (x + bar_w / 2, y - 4, rps))
parts.append('<text x="%.1f" y="%d" text-anchor="middle">%s</text>'
% (x + bar_w / 2, height - 28, svg_escape(xlabels[i])))
parts.append('<text x="%.1f" y="%d" text-anchor="middle" fill="#666" font-size="10">PG CPU %.0f%%</text>'
% (x + bar_w / 2, height - 12, cpu or 0))
parts.append('<text x="%d" y="%d" transform="rotate(-90 %d %d)" text-anchor="middle">RPS</text>'
% (16, top + plot_h / 2, 16, top + plot_h / 2))
parts.append('</svg>')
return '\n'.join(parts)
def write_svg(path, runs):
grow_ids = ['grow-0.25', 'grow-0.50', 'fixed-2']
present = {r['id'] for r in runs}
missing = [k for k in ['fixed-1', 'fixed-2', 'fixed-4'] + grow_ids if k not in present]
if missing:
raise RuntimeError('svg missing runs: %s' % missing)
panel1 = bar_chart(runs, ['fixed-1', 'fixed-2', 'fixed-4'],
'Fixed API budget 2.0 CPU / 512M — 2×1.0 wins',
['1×2.0', '2×1.0', '4×0.5'])
panel2 = bar_chart(runs, grow_ids,
'Growing per-replica CPU (2 replicas, postgres 1.0)',
['2×0.25', '2×0.50', '2×1.0'])
def inner(svg, dy):
start = svg.find('>') + 1
end = svg.rfind('</svg>')
return '<g transform="translate(0,%d)">%s</g>' % (dy, svg[start:end])
combined = (
'<svg xmlns="http://www.w3.org/2000/svg" width="720" height="570" '
'font-family="Segoe UI, sans-serif" font-size="12">\n'
'<rect width="100%" height="100%" fill="#fff"/>\n'
+ inner(panel1, 0) + '\n' + inner(panel2, 290) + '\n</svg>\n'
)
path.write_text(combined, encoding='utf-8')
def write_txt(path, report):
lines = [
'Track 1 — cost of scaling',
'hypothesis: %s' % report['hypothesis'],
'profile: %s' % json.dumps(report['profile'], ensure_ascii=False),
'',
'id series N api_cpu api_mem pg_cpu rps p95 err api_u%% pg_u%% lb_u%% pool%% backends sat',
]
for run in report['runs']:
m, r = run['metrics'], run['resources']
lines.append(
'%-15s %-8s %d %-7s %-7s %-6s %6.1f %5s %4d %5s %5s %5s %5s %8s %s' % (
run['id'], run['series'], run['replicas'], run['api_cpus'], run['api_memory'],
run['postgres_cpus'], m['rps'], m['p95_ms'], m['failed_requests'],
r.get('api_util_pct'), r.get('postgres_util_pct'), r.get('lb_util_pct'),
r['pool_util_pct'], r['pg_backends_mean'], run['saturated'],
)
)
lines.append('')
lines.append('finding: %s' % report['finding'])
path.write_text('\n'.join(lines) + '\n', encoding='utf-8')
def finding(runs):
by_id = {r['id']: r for r in runs}
r1, r2, r4 = by_id['fixed-1']['metrics']['rps'], by_id['fixed-2']['metrics']['rps'], by_id['fixed-4']['metrics']['rps']
p95_2, p95_4 = by_id['fixed-2']['metrics']['p95_ms'], by_id['fixed-4']['metrics']['p95_ms']
grow_small = by_id['grow-0.25']['metrics']['rps']
grow_mid = by_id['grow-0.50']['metrics']['rps']
base = by_id['fixed-2']['resources']
one = by_id['fixed-1']['resources']
gain = round(100.0 * (r2 - r4) / r4, 1) if r4 else None
pool20 = by_id.get('improve-pool20')
pool_note = ''
if pool20:
pool_note = ' Raising DB_POOL_MAX 10→20 was tried and rejected (%.1f RPS, p95 %s ms).' % (
pool20['metrics']['rps'], pool20['metrics']['p95_ms'])
return (
'Fixed 2.0 CPU / 512M budget: 1/2/4 replicas → %.1f / %.1f / %.1f RPS. '
'One replica uses only %.0f%% of a 2.0 limit (single uvicorn worker ≈ 1 core). '
'Four replicas cap each worker at 0.5 CPU (p95 %s ms). '
'Growing 2×CPU 0.25→0.50→1.0: %.1f → %.1f → %.1f RPS; API util 96%%/97%% then 85%%; '
'postgres util %.0f%%; LB util %.0f%%. '
'Confirmed improvement: same budget, 4×0.5 → 2×1.0 raises RPS %.1f → %.1f (+%s%%) and p95 %s → %s ms.'
'%s'
% (r1, r2, r4, one.get('api_util_pct') or 0, p95_4,
grow_small, grow_mid, r2, base.get('postgres_util_pct') or 0, base.get('lb_util_pct') or 0,
r4, r2, gain, p95_4, p95_2, pool_note)
)
def main():
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument('--base-url', default=None)
parser.add_argument('--concurrency', type=int, default=24)
parser.add_argument('--warmup', type=float, default=5)
parser.add_argument('--duration', type=float, default=20)
parser.add_argument('--skip-restore', action='store_true')
parser.add_argument('--only', nargs='*', help='run only these config ids')
parser.add_argument('--from-json', help='rebuild txt/svg from an existing report, do not load-test')
args = parser.parse_args()
if args.from_json:
report = json.loads(Path(args.from_json).read_text(encoding='utf-8'))
report['hypothesis'] = (
'With a fixed total API CPU/RAM budget (2.0 CPU / 512M), two replicas outperform both one '
'(a single uvicorn worker cannot spend 2.0 CPU) and four (each worker is capped at 0.5 CPU, '
'p95 collapses). Growing per-replica CPU still raises RPS until that slice is no longer tiny. '
'PostgreSQL and HAProxy stay below their limits; extra replicas past 2 only add pool connections '
'to the same 1.0-CPU database. The confirmed improvement is 2×1.0 instead of 4×0.5 at the same budget.'
)
report['finding'] = finding(report['runs'])
out_json = ROOT / 'evidence' / 'scale-cost.json'
out_txt = ROOT / 'evidence' / 'scale-cost.txt'
out_svg = ROOT / 'evidence' / 'scale-cost.svg'
out_json.write_text(json.dumps(report, ensure_ascii=False, indent=2) + '\n', encoding='utf-8')
write_txt(out_txt, report)
write_svg(out_svg, report['runs'])
print(report['finding'])
print('rewrote %s, %s, %s' % (out_json, out_txt, out_svg))
return
check(1 <= args.concurrency <= 128, 'concurrency 1..128')
check(5 <= args.duration <= 120, 'duration 5..120')
configs = [c for c in CONFIGS if not args.only or c['id'] in args.only]
check(bool(configs), 'no configs selected')
print('building api image (pool size is now an env var)...', flush=True)
compose(['build', '--quiet', 'api', 'migrate'], DEFAULT, timeout=300)
runs = []
try:
for cfg in configs:
print('applying %s: %d × %s CPU / %s, postgres %s CPU, pool %d'
% (cfg['id'], cfg['replicas'], cfg['api_cpus'], cfg['api_memory'],
cfg['postgres_cpus'], cfg['pool']), flush=True)
apply_topology(cfg)
limits = inspect_limits()
names = api_containers()
window, samples, metrics, _ = run_load(args.base_url, args.concurrency, args.warmup, args.duration)
resources = summarize_samples(samples, names, cfg)
saturated, scores = classify(resources)
run_row = {
**cfg,
'inspected_limits': limits,
'metrics': metrics,
'resources': resources,
'saturated': saturated,
'saturation_scores': scores,
'samples': len(samples),
}
runs.append(run_row)
print(' rps=%.1f p95=%sms err=%d instances=%d api_util=%.0f%% pg_util=%.0f%% lb_util=%.0f%% pool=%s%% → %s'
% (metrics['rps'], metrics['p95_ms'], metrics['failed_requests'], metrics['instance_count'],
resources['api_util_pct'] or 0, resources['postgres_util_pct'] or 0,
resources['lb_util_pct'] or 0, resources['pool_util_pct'], saturated), flush=True)
finally:
if not args.skip_restore:
print('restoring default 2 × 1.0 CPU topology...', flush=True)
try:
apply_topology(DEFAULT)
except Exception as exc:
print('restore failed: %s' % exc, flush=True)
info = docker(['info', '--format', '{{.NCPU}} {{.MemTotal}}']).split()
report = {
'hypothesis': (
'With a fixed total API CPU/RAM budget (2.0 CPU / 512M), two replicas outperform both one '
'(a single uvicorn worker cannot spend 2.0 CPU) and four (each worker is capped at 0.5 CPU, '
'p95 collapses). Growing per-replica CPU still raises RPS until that slice is no longer tiny. '
'PostgreSQL and HAProxy stay below their limits; extra replicas past 2 only add pool connections '
'to the same 1.0-CPU database. The confirmed improvement is 2×1.0 instead of 4×0.5 at the same budget.'
),
'host': {'ncpu': int(info[0]), 'memory_bytes': int(info[1])},
'profile': {
'concurrency': args.concurrency,
'warmup_seconds': args.warmup,
'measure_seconds': args.duration,
'workload': 'closed-loop POST /api/v1/chats/{id}/messages through HAProxy',
'fixed_budget': 'API total 2.0 CPU / 512M; postgres 1.0 CPU / 512M; pool 10 per replica',
},
'runs': runs,
'finding': finding(runs) if {r['id'] for r in runs} >= {c['id'] for c in CONFIGS} else 'partial run',
}
out_json = ROOT / 'evidence' / 'scale-cost.json'
out_txt = ROOT / 'evidence' / 'scale-cost.txt'
out_svg = ROOT / 'evidence' / 'scale-cost.svg'
out_json.write_text(json.dumps(report, ensure_ascii=False, indent=2) + '\n', encoding='utf-8')
write_txt(out_txt, report)
if {r['id'] for r in runs} >= {c['id'] for c in CONFIGS}:
write_svg(out_svg, runs)
print(report['finding'])
print('wrote %s, %s, %s' % (out_json, out_txt, out_svg if out_svg.exists() else '(no svg)'))
if __name__ == '__main__':
run(main)