Files
mtusi2026lab3/scripts/scaling.py
T

151 lines
7.4 KiB
Python

"""Controlled 1/2/4 replica study, resource samples, raw requests, pool intervention.
Synthetic data only. Same 20% POST / 80% first history page workload in every run.
Short closed-loop runs measure local HTTP confirmations, not a production capacity SLA.
"""
import argparse
import asyncio
from collections import Counter
import csv
import json
import statistics
import threading
import time
import uuid
import httpx
from stack import ROOT, docker, ready, replicas
OUT = ROOT / 'evidence/scaling'
OUT.mkdir(parents=True, exist_ok=True)
def setup(n, cpu, memory, pool):
# Temporary ignored override contains no credentials; base configuration is restored in finally.
override = ROOT / '.env.scaling.json'
override.write_text(json.dumps({'services': {'api': {'cpus': cpu, 'mem_limit': str(memory) + 'm',
'environment': {'DB_POOL_SIZE': str(pool)}}}}), encoding='utf-8')
docker('compose', '-f', 'compose.yaml', '-f', str(override), 'up', '-d', '--no-deps', '--wait', '--scale', f'api={n}', 'api')
with httpx.Client(base_url='http://127.0.0.1:8080', timeout=10) as c:
ready(c)
deadline = time.monotonic() + 25
while time.monotonic() < deadline:
seen = {c.get('/health/ready').headers.get('X-Instance-Id') for _ in range(40)} - {None}
if len(seen) == n:
break
time.sleep(.5)
else:
raise AssertionError(f'LB sees {len(seen)}, expected {n}')
return [{'id': cid, 'limits': json.loads(docker('inspect', '--format',
'{"nano_cpus":{{.HostConfig.NanoCpus}},"memory_bytes":{{.HostConfig.Memory}}}', cid))}
for cid in replicas()]
def sample(stop, samples):
while not stop.is_set():
tick = time.time()
stats = docker('compose', 'ps', '-q', 'api', 'db', 'cache', 'lb').splitlines()
resources = docker('stats', '--no-stream', '--format', '{{json .}}', *stats)
waits = docker('compose', 'exec', '-T', 'db', 'psql', '-U', 'messenger', '-d', 'messenger', '-Atc',
"SELECT json_build_object('active',count(*) FILTER (WHERE state='active'),"
"'idle_transaction',count(*) FILTER (WHERE state='idle in transaction'),"
"'lock_wait',count(*) FILTER (WHERE wait_event_type='Lock')) FROM pg_stat_activity WHERE backend_type='client backend'")
samples.append({'unix_time': tick, 'containers': [json.loads(s) for s in resources.splitlines()],
'database': json.loads(waits)})
stop.wait(.5)
async def traffic(path, headers, concurrency, duration, record=True):
rows = []
start = time.perf_counter()
async with httpx.AsyncClient(base_url='http://127.0.0.1:8080', timeout=10,
limits=httpx.Limits(max_connections=concurrency, max_keepalive_connections=concurrency)) as c:
async def worker(i):
seq = i
while time.perf_counter() - start < duration:
method = 'POST' if seq % 5 == 0 else 'GET'
tick = time.perf_counter()
try:
if method == 'POST':
r = await c.post(path, headers={**headers, 'Idempotency-Key': uuid.uuid4().hex}, json={'text': 'x' * 1000})
else:
r = await c.get(path, headers=headers, params={'limit': 50})
code, serving = r.status_code, r.headers.get('X-Instance-Id', '')
except httpx.HTTPError:
code, serving = 0, ''
if record:
rows.append((round(tick - start, 6), method, code, round((time.perf_counter() - tick) * 1000, 3), serving))
seq += 1
await asyncio.gather(*(worker(i) for i in range(concurrency)))
return rows, time.perf_counter() - start
def run(label, concurrency, repeat, duration):
with httpx.Client(base_url='http://127.0.0.1:8080', timeout=12) as c:
name = 'load_' + uuid.uuid4().hex[:20]
password = 'load-password-only-123'
r = c.post('/api/v1/users', json={'username': name, 'display_name': 'Load', 'password': password})
assert r.status_code == 201, r.text
uid = r.json()['id']
r = c.post('/api/v1/auth/sessions', auth=(name, password))
assert r.status_code == 201
headers = {'Authorization': 'Bearer ' + r.json()['token']}
r = c.post('/api/v1/chats', headers=headers, json={'title': 'Load', 'member_ids': [uid]})
assert r.status_code == 201
path = '/api/v1/chats/' + r.json()['id'] + '/messages'
for _ in range(60):
r = c.post(path, headers={**headers, 'Idempotency-Key': uuid.uuid4().hex}, json={'text': 'x' * 1000})
assert r.status_code == 201
asyncio.run(traffic(path, headers, concurrency, 2, record=False))
samples, stop = [], threading.Event()
thread = threading.Thread(target=sample, args=(stop, samples))
thread.start()
client_cpu_start = time.process_time()
try:
rows, elapsed = asyncio.run(traffic(path, headers, concurrency, duration))
finally:
stop.set()
thread.join()
slug = f'{label}-c{concurrency}-r{repeat}'
with (OUT / (slug + '.csv')).open('w', newline='', encoding='utf-8') as f:
writer = csv.writer(f)
writer.writerow(['start_seconds', 'method', 'status', 'latency_ms', 'instance'])
writer.writerows(rows)
(OUT / (slug + '-resources.json')).write_text(json.dumps(samples, indent=2) + '\n', encoding='utf-8')
latencies = sorted(r[3] for r in rows)
good = [r for r in rows if r[2] in (200, 201)]
result = {'label': label, 'concurrency': concurrency, 'repeat': repeat, 'elapsed': round(elapsed, 3),
'client_cpu_seconds': round(time.process_time() - client_cpu_start, 3),
'attempts': len(rows), 'successful_rps': round(len(good) / elapsed, 3),
'status_counts': dict(Counter(str(r[2]) for r in rows)), 'errors': len(rows) - len(good),
'p50_ms': statistics.median(latencies), 'p95_ms': latencies[int(.95 * (len(latencies) - 1))],
'p99_ms': latencies[int(.99 * (len(latencies) - 1))], 'instances': sorted({r[4] for r in good})}
print(json.dumps(result), flush=True)
return result
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--duration', type=int, default=10)
parser.add_argument('--repeats', type=int, default=2)
args = parser.parse_args()
results, configurations = [], []
cases = [(f'fixed-{n}', n, 1 / n, 768 // n, 5) for n in (1, 2, 4)]
cases += [(f'growing-{n}', n, .5, 192, 5) for n in (1, 2, 4)]
cases += [('pool1-2', 2, .5, 192, 1)]
try:
for label, n, cpu, memory, pool in cases:
actual = setup(n, cpu, memory, pool)
configurations.append({'label': label, 'replicas': n, 'cpu_each': cpu, 'memory_mib_each': memory,
'pool_each': pool, 'actual_containers': actual})
for concurrency in (8, 24, 64):
for repeat in range(1, args.repeats + 1):
results.append(run(label, concurrency, repeat, args.duration))
(OUT / 'summary.json').write_text(json.dumps({'configurations': configurations, 'results': results}, indent=2) + '\n', encoding='utf-8')
finally:
docker('compose', 'up', '-d', '--no-deps', '--wait', '--scale', 'api=2', 'api')
if __name__ == '__main__':
main()