forked from mtusi2026/mtusi2026lab3
244 lines
12 KiB
Python
244 lines
12 KiB
Python
"""Repeatable LB-only 80% history reads / 20% unique message writes, closed loop."""
|
|
import argparse
|
|
import collections
|
|
import concurrent.futures
|
|
import http.client
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import platform
|
|
import resource
|
|
import statistics
|
|
import subprocess
|
|
import sys
|
|
import threading
|
|
import time
|
|
import uuid
|
|
from urllib.parse import urlsplit
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT / 'tests'))
|
|
from client import Client
|
|
|
|
|
|
def command(args, env=None):
|
|
return subprocess.check_output(args, text=True, env=env).strip()
|
|
|
|
|
|
def containers():
|
|
ids = command(['docker', 'compose', 'ps', '-q']).split()
|
|
result = {}
|
|
for cid in ids:
|
|
role = command(['docker', 'inspect', '--format', '{{index .Config.Labels "com.docker.compose.service"}}', cid])
|
|
result[cid] = role
|
|
return result
|
|
|
|
|
|
def telemetry(done, result):
|
|
roles = containers()
|
|
while not done.is_set():
|
|
started = time.monotonic()
|
|
raw = command(['docker', 'stats', '--no-stream', '--format', '{{json .}}', *roles])
|
|
sample = []
|
|
for line in raw.splitlines():
|
|
value = json.loads(line)
|
|
role = next((role for cid, role in roles.items() if cid.startswith(value['ID'])), '?')
|
|
sample.append({'service': role, 'container': value['ID'], 'cpu_percent': float(value['CPUPerc'].rstrip('%')), 'memory': value['MemUsage']})
|
|
waits = command(['docker', 'compose', 'exec', '-T', 'postgres', 'psql', '-U', 'messenger', '-d', 'messenger', '-Atc', "SELECT coalesce(json_agg(s), '[]') FROM (SELECT state, wait_event_type, wait_event, count(*) AS connections FROM pg_stat_activity WHERE 1=1 AND datname = 'messenger' AND pid <> pg_backend_pid() GROUP BY state, wait_event_type, wait_event) s"])
|
|
result.append({'t_monotonic': started, 'containers': sample, 'postgres_activity': json.loads(waits)})
|
|
done.wait(.1)
|
|
|
|
|
|
def fixture():
|
|
# An overloaded LB needs time to mark recovered backends UP before seeding.
|
|
c = Client(timeout=2)
|
|
for attempt in range(120):
|
|
try:
|
|
c.call('GET', '/health/ready', schema='Health')
|
|
break
|
|
except Exception:
|
|
time.sleep(.25)
|
|
else:
|
|
raise AssertionError('LB did not recover before fixture')
|
|
time.sleep(1)
|
|
actor = c.user('bench')
|
|
chats = [c.call('POST', '/api/v1/chats', 201, actor=actor, body={'title': 'synthetic-benchmark', 'member_ids': [actor['id']]}, schema='Chat')['json'] for _ in range(8)]
|
|
# Seed outside the measurement with the same 800-message fixture each run.
|
|
payload = json.dumps({'user': actor['id'], 'chats': [v['id'] for v in chats]})
|
|
code = """import json, sys, uuid, psycopg
|
|
sys.path.insert(0, '/app/src')
|
|
from postgres import setting
|
|
p = json.loads(sys.stdin.read())
|
|
with psycopg.connect(setting('DATABASE_URL')) as conn:
|
|
for chat in p['chats']:
|
|
conn.execute(\"INSERT INTO messages (id, chat_id, sender_id, text, created_at) SELECT gen_random_uuid(), %s::uuid, %s::uuid, convert_to(repeat('x', 64), 'UTF8'), TIMESTAMPTZ '2026-01-01 00:00:00Z' + n * INTERVAL '1 microsecond' FROM generate_series(1, 100) n\", (chat, p['user']))
|
|
print('Seeded 800 synthetic messages.')
|
|
"""
|
|
subprocess.run(['docker', 'compose', 'exec', '-T', 'api', 'python', '-c', code], input=payload, text=True, check=True, stdout=subprocess.DEVNULL)
|
|
return actor, chats
|
|
|
|
|
|
def percentile(values, fraction):
|
|
return values[max(0, int(len(values) * fraction + .999999) - 1)] if values else None
|
|
|
|
|
|
def measure(actor, chats, concurrency, duration):
|
|
parsed = urlsplit(os.environ.get('BASE_URL', 'http://127.0.0.1:18087'))
|
|
assert parsed.scheme == 'http', 'local synthetic stand only'
|
|
gate = threading.Barrier(concurrency + 1)
|
|
deadline = [0.0]
|
|
def worker(index):
|
|
connection = http.client.HTTPConnection(parsed.hostname, parsed.port, timeout=3)
|
|
headers = {'Authorization': 'Bearer ' + actor['token'], 'Content-Type': 'application/json'}
|
|
times, errors, seen = [], collections.Counter(), collections.Counter()
|
|
reads = writes = 0
|
|
step = index
|
|
gate.wait()
|
|
try:
|
|
while time.monotonic() < deadline[0]:
|
|
chat = chats[(step // 5 + index) % len(chats)]
|
|
path = '/api/v1/chats/' + chat['id'] + '/messages'
|
|
write = step % 5 == 0
|
|
body = '{"text":"' + 'x' * 64 + '"}' if write else None
|
|
request_headers = dict(headers)
|
|
if write:
|
|
request_headers['Idempotency-Key'] = uuid.uuid4().hex
|
|
else:
|
|
path += '?limit=50'
|
|
start = time.monotonic()
|
|
try:
|
|
connection.request('POST' if write else 'GET', path, body=body, headers=request_headers)
|
|
response = connection.getresponse()
|
|
raw = response.read()
|
|
if response.status == (201 if write else 200):
|
|
payload = json.loads(raw)
|
|
assert ('id' in payload) if write else (len(payload['items']) == 50)
|
|
times.append((time.monotonic() - start) * 1000)
|
|
seen[response.getheader('X-Instance-Id')] += 1
|
|
writes += int(write)
|
|
reads += int(not write)
|
|
else:
|
|
errors['http_' + str(response.status)] += 1
|
|
except (OSError, http.client.HTTPException) as exc:
|
|
errors[type(exc).__name__] += 1
|
|
connection.close()
|
|
step += 1
|
|
finally:
|
|
connection.close()
|
|
return times, errors, seen, reads, writes
|
|
cpu_before = resource.getrusage(resource.RUSAGE_SELF)
|
|
with concurrent.futures.ThreadPoolExecutor(max_workers=concurrency) as executor:
|
|
futures = [executor.submit(worker, i) for i in range(concurrency)]
|
|
started = time.monotonic()
|
|
deadline[0] = started + duration
|
|
gate.wait()
|
|
rows = [f.result() for f in futures]
|
|
elapsed = time.monotonic() - started
|
|
cpu_after = resource.getrusage(resource.RUSAGE_SELF)
|
|
times = sorted(t for row in rows for t in row[0])
|
|
errors, instances = collections.Counter(), collections.Counter()
|
|
for row in rows:
|
|
errors.update(row[1]); instances.update(row[2])
|
|
count = len(times)
|
|
return {'duration_seconds': elapsed, 'configured_seconds': duration, 'concurrency': concurrency, 'successes': count, 'reads': sum(v[3] for v in rows), 'writes': sum(v[4] for v in rows), 'successful_rps': count / elapsed, 'errors': dict(errors), 'error_fraction': sum(errors.values()) / max(1, count + sum(errors.values())), 'p50_ms': percentile(times, .5), 'p95_ms': percentile(times, .95), 'p99_ms': percentile(times, .99), 'instances': dict(instances), 'generator_cpu_percent': ((cpu_after.ru_utime + cpu_after.ru_stime) - (cpu_before.ru_utime + cpu_before.ru_stime)) / elapsed * 100}
|
|
|
|
|
|
def pool_snapshot():
|
|
result = {}
|
|
for cid in command(['docker', 'compose', 'ps', '-q', 'api']).split():
|
|
command(['docker', 'kill', '--signal=USR1', cid])
|
|
logs = subprocess.check_output(['docker', 'logs', '--tail', '10', cid], text=True, stderr=subprocess.STDOUT)
|
|
lines = [line for line in logs.splitlines() if 'pool_stats instance=' in line]
|
|
assert lines, 'pool diagnostics signal missing'
|
|
stats = json.loads(lines[-1].split('stats=', 1)[1])
|
|
code = "from pathlib import Path; import json; print(json.dumps(dict(line.split() for line in Path('/sys/fs/cgroup/cpu.stat').read_text().splitlines())))"
|
|
stats['cgroup_cpu'] = json.loads(command(['docker', 'exec', cid, 'python', '-c', code]))
|
|
result[cid[:12]] = stats
|
|
return result
|
|
|
|
|
|
def trial(concurrency, duration):
|
|
actor, chats = fixture()
|
|
measure(actor, chats, concurrency, 2)
|
|
before = pool_snapshot()
|
|
samples = []
|
|
done = threading.Event()
|
|
with concurrent.futures.ThreadPoolExecutor(max_workers=1) as executor:
|
|
probe = executor.submit(telemetry, done, samples)
|
|
try:
|
|
result = measure(actor, chats, concurrency, duration)
|
|
finally:
|
|
done.set()
|
|
probe.result()
|
|
result['pool_before'] = before
|
|
result['pool_after'] = pool_snapshot()
|
|
result['telemetry'] = samples
|
|
result['fixture'] = {'chats': 8, 'messages_before_warmup': 800, 'text_code_points': 64, 'history_limit': 50, 'warmup_seconds': 2, 'write_ratio_target': .2}
|
|
return result
|
|
|
|
|
|
def configure(replicas, cpu, memory, mode, pool=8, database_cpu=1):
|
|
env = dict(os.environ, API_REPLICAS=str(replicas), API_CPUS=str(cpu), API_MEMORY=str(memory) + 'm', DB_ACCESS_MODE=mode, DB_POOL_MAX=str(pool), POSTGRES_CPUS=str(database_cpu))
|
|
subprocess.run(['docker', 'compose', 'up', '-d', '--wait', '--no-deps', 'postgres'], env=env, check=True, stdout=subprocess.DEVNULL)
|
|
# Independent runs use a distinct new fixture; application replicas are recreated.
|
|
subprocess.run(['docker', 'compose', 'up', '-d', '--wait', '--no-deps', '--scale', 'api=' + str(replicas), 'api'], env=env, check=True, stdout=subprocess.DEVNULL)
|
|
c = Client(timeout=1)
|
|
deadline = time.monotonic() + 30
|
|
observed = set()
|
|
while time.monotonic() < deadline:
|
|
try:
|
|
observed.add(c.call('GET', '/health/ready', schema='Health')['headers']['X-Instance-Id'])
|
|
if len(observed) >= replicas:
|
|
break
|
|
except Exception:
|
|
pass
|
|
time.sleep(.1)
|
|
assert len(observed) == replicas
|
|
time.sleep(2)
|
|
return env
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument('--duration', type=float, default=8)
|
|
parser.add_argument('--repeats', type=int, default=3)
|
|
parser.add_argument('--pilot', action='store_true')
|
|
parser.add_argument('--resume', action='store_true')
|
|
args = parser.parse_args()
|
|
output = ROOT / 'evidence/scaling.json'
|
|
result = {'runtime': platform.python_version(), 'host': platform.platform(), 'docker': json.loads(command(['docker', 'info', '--format', '{"cpus":{{.NCPU}},"memory":{{.MemTotal}},"os":"{{.OperatingSystem}}"}'])), 'runs': []}
|
|
configs = [('fixed', n, 1 / n, 512 // n, 'optimized', 16) for n in (1, 2, 4)]
|
|
configs += [('growing', n, .5, 256, 'optimized', 16) for n in (1, 2, 4)]
|
|
configs += [('improvement', 2, .5, 256, 'baseline', 16)]
|
|
configs += [('saturation', 4, .5, 256, 'optimized', n) for n in (1, 4, 32, 64)]
|
|
configs = [(*row, 8) for row in configs]
|
|
configs += [('pool_limit', 1, 1, 512, 'optimized', 32, n) for n in (1, 8)]
|
|
configs += [('pool_isolated', 1, 1, 512, 'optimized', 16, n) for n in (1,8)]
|
|
configs = [(*row, 1) for row in configs]
|
|
configs += [('database_limit', n, .5, 256, 'optimized', 16, 8, .1) for n in (1,2,4)]
|
|
if args.resume:
|
|
result = json.loads(output.read_text())
|
|
if args.pilot:
|
|
configs = [('pilot', 2, .5, 256, 'optimized', 16, 8, 1)]
|
|
try:
|
|
for series, replicas, cpu, memory, mode, concurrency, pool, database_cpu in configs:
|
|
completed = [v for v in result['runs'] if v['series'] == series and v['replicas'] == replicas and v['concurrency'] == concurrency and v['db_pool_max'] == pool]
|
|
if len(completed) >= args.repeats:
|
|
continue
|
|
configure(replicas, cpu, memory, mode, pool, database_cpu)
|
|
for repeat in range(args.repeats):
|
|
if any(v['series'] == series and v['replicas'] == replicas and v['concurrency'] == concurrency and v['db_pool_max'] == pool and v['repeat'] == repeat + 1 for v in result['runs']):
|
|
continue
|
|
row = trial(concurrency, args.duration)
|
|
row.update(series=series, replicas=replicas, api_cpu_each=cpu, api_memory_mib_each=memory, mode=mode, repeat=repeat + 1, db_pool_max=pool, postgres_cpu=database_cpu)
|
|
result['runs'].append(row)
|
|
output.write_text(json.dumps(result, indent=2))
|
|
print(json.dumps({k: row[k] for k in ('series', 'replicas', 'mode', 'repeat', 'concurrency', 'successful_rps', 'p95_ms', 'errors', 'generator_cpu_percent')}), flush=True)
|
|
finally:
|
|
configure(2, 1, 256, 'optimized')
|
|
print('Research complete; default two-replica stand restored.')
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|