151 lines
7.4 KiB
Python
151 lines
7.4 KiB
Python
"""Controlled 1/2/4 replica study, resource samples, raw requests, pool intervention.
|
|
|
|
Synthetic data only. Same 20% POST / 80% first history page workload in every run.
|
|
Short closed-loop runs measure local HTTP confirmations, not a production capacity SLA.
|
|
"""
|
|
import argparse
|
|
import asyncio
|
|
from collections import Counter
|
|
import csv
|
|
import json
|
|
import statistics
|
|
import threading
|
|
import time
|
|
import uuid
|
|
|
|
import httpx
|
|
from stack import ROOT, docker, ready, replicas
|
|
|
|
OUT = ROOT / 'evidence/scaling'
|
|
OUT.mkdir(parents=True, exist_ok=True)
|
|
|
|
|
|
def setup(n, cpu, memory, pool):
|
|
# Temporary ignored override contains no credentials; base configuration is restored in finally.
|
|
override = ROOT / '.env.scaling.json'
|
|
override.write_text(json.dumps({'services': {'api': {'cpus': cpu, 'mem_limit': str(memory) + 'm',
|
|
'environment': {'DB_POOL_SIZE': str(pool)}}}}), encoding='utf-8')
|
|
docker('compose', '-f', 'compose.yaml', '-f', str(override), 'up', '-d', '--no-deps', '--wait', '--scale', f'api={n}', 'api')
|
|
with httpx.Client(base_url='http://127.0.0.1:8080', timeout=10) as c:
|
|
ready(c)
|
|
deadline = time.monotonic() + 25
|
|
while time.monotonic() < deadline:
|
|
seen = {c.get('/health/ready').headers.get('X-Instance-Id') for _ in range(40)} - {None}
|
|
if len(seen) == n:
|
|
break
|
|
time.sleep(.5)
|
|
else:
|
|
raise AssertionError(f'LB sees {len(seen)}, expected {n}')
|
|
return [{'id': cid, 'limits': json.loads(docker('inspect', '--format',
|
|
'{"nano_cpus":{{.HostConfig.NanoCpus}},"memory_bytes":{{.HostConfig.Memory}}}', cid))}
|
|
for cid in replicas()]
|
|
|
|
|
|
def sample(stop, samples):
|
|
while not stop.is_set():
|
|
tick = time.time()
|
|
stats = docker('compose', 'ps', '-q', 'api', 'db', 'cache', 'lb').splitlines()
|
|
resources = docker('stats', '--no-stream', '--format', '{{json .}}', *stats)
|
|
waits = docker('compose', 'exec', '-T', 'db', 'psql', '-U', 'messenger', '-d', 'messenger', '-Atc',
|
|
"SELECT json_build_object('active',count(*) FILTER (WHERE state='active'),"
|
|
"'idle_transaction',count(*) FILTER (WHERE state='idle in transaction'),"
|
|
"'lock_wait',count(*) FILTER (WHERE wait_event_type='Lock')) FROM pg_stat_activity WHERE backend_type='client backend'")
|
|
samples.append({'unix_time': tick, 'containers': [json.loads(s) for s in resources.splitlines()],
|
|
'database': json.loads(waits)})
|
|
stop.wait(.5)
|
|
|
|
|
|
async def traffic(path, headers, concurrency, duration, record=True):
|
|
rows = []
|
|
start = time.perf_counter()
|
|
async with httpx.AsyncClient(base_url='http://127.0.0.1:8080', timeout=10,
|
|
limits=httpx.Limits(max_connections=concurrency, max_keepalive_connections=concurrency)) as c:
|
|
async def worker(i):
|
|
seq = i
|
|
while time.perf_counter() - start < duration:
|
|
method = 'POST' if seq % 5 == 0 else 'GET'
|
|
tick = time.perf_counter()
|
|
try:
|
|
if method == 'POST':
|
|
r = await c.post(path, headers={**headers, 'Idempotency-Key': uuid.uuid4().hex}, json={'text': 'x' * 1000})
|
|
else:
|
|
r = await c.get(path, headers=headers, params={'limit': 50})
|
|
code, serving = r.status_code, r.headers.get('X-Instance-Id', '')
|
|
except httpx.HTTPError:
|
|
code, serving = 0, ''
|
|
if record:
|
|
rows.append((round(tick - start, 6), method, code, round((time.perf_counter() - tick) * 1000, 3), serving))
|
|
seq += 1
|
|
await asyncio.gather(*(worker(i) for i in range(concurrency)))
|
|
return rows, time.perf_counter() - start
|
|
|
|
|
|
def run(label, concurrency, repeat, duration):
|
|
with httpx.Client(base_url='http://127.0.0.1:8080', timeout=12) as c:
|
|
name = 'load_' + uuid.uuid4().hex[:20]
|
|
password = 'load-password-only-123'
|
|
r = c.post('/api/v1/users', json={'username': name, 'display_name': 'Load', 'password': password})
|
|
assert r.status_code == 201, r.text
|
|
uid = r.json()['id']
|
|
r = c.post('/api/v1/auth/sessions', auth=(name, password))
|
|
assert r.status_code == 201
|
|
headers = {'Authorization': 'Bearer ' + r.json()['token']}
|
|
r = c.post('/api/v1/chats', headers=headers, json={'title': 'Load', 'member_ids': [uid]})
|
|
assert r.status_code == 201
|
|
path = '/api/v1/chats/' + r.json()['id'] + '/messages'
|
|
for _ in range(60):
|
|
r = c.post(path, headers={**headers, 'Idempotency-Key': uuid.uuid4().hex}, json={'text': 'x' * 1000})
|
|
assert r.status_code == 201
|
|
asyncio.run(traffic(path, headers, concurrency, 2, record=False))
|
|
samples, stop = [], threading.Event()
|
|
thread = threading.Thread(target=sample, args=(stop, samples))
|
|
thread.start()
|
|
client_cpu_start = time.process_time()
|
|
try:
|
|
rows, elapsed = asyncio.run(traffic(path, headers, concurrency, duration))
|
|
finally:
|
|
stop.set()
|
|
thread.join()
|
|
slug = f'{label}-c{concurrency}-r{repeat}'
|
|
with (OUT / (slug + '.csv')).open('w', newline='', encoding='utf-8') as f:
|
|
writer = csv.writer(f)
|
|
writer.writerow(['start_seconds', 'method', 'status', 'latency_ms', 'instance'])
|
|
writer.writerows(rows)
|
|
(OUT / (slug + '-resources.json')).write_text(json.dumps(samples, indent=2) + '\n', encoding='utf-8')
|
|
latencies = sorted(r[3] for r in rows)
|
|
good = [r for r in rows if r[2] in (200, 201)]
|
|
result = {'label': label, 'concurrency': concurrency, 'repeat': repeat, 'elapsed': round(elapsed, 3),
|
|
'client_cpu_seconds': round(time.process_time() - client_cpu_start, 3),
|
|
'attempts': len(rows), 'successful_rps': round(len(good) / elapsed, 3),
|
|
'status_counts': dict(Counter(str(r[2]) for r in rows)), 'errors': len(rows) - len(good),
|
|
'p50_ms': statistics.median(latencies), 'p95_ms': latencies[int(.95 * (len(latencies) - 1))],
|
|
'p99_ms': latencies[int(.99 * (len(latencies) - 1))], 'instances': sorted({r[4] for r in good})}
|
|
print(json.dumps(result), flush=True)
|
|
return result
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument('--duration', type=int, default=10)
|
|
parser.add_argument('--repeats', type=int, default=2)
|
|
args = parser.parse_args()
|
|
results, configurations = [], []
|
|
cases = [(f'fixed-{n}', n, 1 / n, 768 // n, 5) for n in (1, 2, 4)]
|
|
cases += [(f'growing-{n}', n, .5, 192, 5) for n in (1, 2, 4)]
|
|
cases += [('pool1-2', 2, .5, 192, 1)]
|
|
try:
|
|
for label, n, cpu, memory, pool in cases:
|
|
actual = setup(n, cpu, memory, pool)
|
|
configurations.append({'label': label, 'replicas': n, 'cpu_each': cpu, 'memory_mib_each': memory,
|
|
'pool_each': pool, 'actual_containers': actual})
|
|
for concurrency in (8, 24, 64):
|
|
for repeat in range(1, args.repeats + 1):
|
|
results.append(run(label, concurrency, repeat, args.duration))
|
|
(OUT / 'summary.json').write_text(json.dumps({'configurations': configurations, 'results': results}, indent=2) + '\n', encoding='utf-8')
|
|
finally:
|
|
docker('compose', 'up', '-d', '--no-deps', '--wait', '--scale', 'api=2', 'api')
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|