Files
mtusi2026lab3/student-tests/benchmark.py

244 lines
12 KiB
Python

"""Repeatable LB-only 80% history reads / 20% unique message writes, closed loop."""
import argparse
import collections
import concurrent.futures
import http.client
import json
import os
from pathlib import Path
import platform
import resource
import statistics
import subprocess
import sys
import threading
import time
import uuid
from urllib.parse import urlsplit
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / 'tests'))
from client import Client
def command(args, env=None):
return subprocess.check_output(args, text=True, env=env).strip()
def containers():
ids = command(['docker', 'compose', 'ps', '-q']).split()
result = {}
for cid in ids:
role = command(['docker', 'inspect', '--format', '{{index .Config.Labels "com.docker.compose.service"}}', cid])
result[cid] = role
return result
def telemetry(done, result):
roles = containers()
while not done.is_set():
started = time.monotonic()
raw = command(['docker', 'stats', '--no-stream', '--format', '{{json .}}', *roles])
sample = []
for line in raw.splitlines():
value = json.loads(line)
role = next((role for cid, role in roles.items() if cid.startswith(value['ID'])), '?')
sample.append({'service': role, 'container': value['ID'], 'cpu_percent': float(value['CPUPerc'].rstrip('%')), 'memory': value['MemUsage']})
waits = command(['docker', 'compose', 'exec', '-T', 'postgres', 'psql', '-U', 'messenger', '-d', 'messenger', '-Atc', "SELECT coalesce(json_agg(s), '[]') FROM (SELECT state, wait_event_type, wait_event, count(*) AS connections FROM pg_stat_activity WHERE 1=1 AND datname = 'messenger' AND pid <> pg_backend_pid() GROUP BY state, wait_event_type, wait_event) s"])
result.append({'t_monotonic': started, 'containers': sample, 'postgres_activity': json.loads(waits)})
done.wait(.1)
def fixture():
# An overloaded LB needs time to mark recovered backends UP before seeding.
c = Client(timeout=2)
for attempt in range(120):
try:
c.call('GET', '/health/ready', schema='Health')
break
except Exception:
time.sleep(.25)
else:
raise AssertionError('LB did not recover before fixture')
time.sleep(1)
actor = c.user('bench')
chats = [c.call('POST', '/api/v1/chats', 201, actor=actor, body={'title': 'synthetic-benchmark', 'member_ids': [actor['id']]}, schema='Chat')['json'] for _ in range(8)]
# Seed outside the measurement with the same 800-message fixture each run.
payload = json.dumps({'user': actor['id'], 'chats': [v['id'] for v in chats]})
code = """import json, sys, uuid, psycopg
sys.path.insert(0, '/app/src')
from postgres import setting
p = json.loads(sys.stdin.read())
with psycopg.connect(setting('DATABASE_URL')) as conn:
for chat in p['chats']:
conn.execute(\"INSERT INTO messages (id, chat_id, sender_id, text, created_at) SELECT gen_random_uuid(), %s::uuid, %s::uuid, convert_to(repeat('x', 64), 'UTF8'), TIMESTAMPTZ '2026-01-01 00:00:00Z' + n * INTERVAL '1 microsecond' FROM generate_series(1, 100) n\", (chat, p['user']))
print('Seeded 800 synthetic messages.')
"""
subprocess.run(['docker', 'compose', 'exec', '-T', 'api', 'python', '-c', code], input=payload, text=True, check=True, stdout=subprocess.DEVNULL)
return actor, chats
def percentile(values, fraction):
return values[max(0, int(len(values) * fraction + .999999) - 1)] if values else None
def measure(actor, chats, concurrency, duration):
parsed = urlsplit(os.environ.get('BASE_URL', 'http://127.0.0.1:18087'))
assert parsed.scheme == 'http', 'local synthetic stand only'
gate = threading.Barrier(concurrency + 1)
deadline = [0.0]
def worker(index):
connection = http.client.HTTPConnection(parsed.hostname, parsed.port, timeout=3)
headers = {'Authorization': 'Bearer ' + actor['token'], 'Content-Type': 'application/json'}
times, errors, seen = [], collections.Counter(), collections.Counter()
reads = writes = 0
step = index
gate.wait()
try:
while time.monotonic() < deadline[0]:
chat = chats[(step // 5 + index) % len(chats)]
path = '/api/v1/chats/' + chat['id'] + '/messages'
write = step % 5 == 0
body = '{"text":"' + 'x' * 64 + '"}' if write else None
request_headers = dict(headers)
if write:
request_headers['Idempotency-Key'] = uuid.uuid4().hex
else:
path += '?limit=50'
start = time.monotonic()
try:
connection.request('POST' if write else 'GET', path, body=body, headers=request_headers)
response = connection.getresponse()
raw = response.read()
if response.status == (201 if write else 200):
payload = json.loads(raw)
assert ('id' in payload) if write else (len(payload['items']) == 50)
times.append((time.monotonic() - start) * 1000)
seen[response.getheader('X-Instance-Id')] += 1
writes += int(write)
reads += int(not write)
else:
errors['http_' + str(response.status)] += 1
except (OSError, http.client.HTTPException) as exc:
errors[type(exc).__name__] += 1
connection.close()
step += 1
finally:
connection.close()
return times, errors, seen, reads, writes
cpu_before = resource.getrusage(resource.RUSAGE_SELF)
with concurrent.futures.ThreadPoolExecutor(max_workers=concurrency) as executor:
futures = [executor.submit(worker, i) for i in range(concurrency)]
started = time.monotonic()
deadline[0] = started + duration
gate.wait()
rows = [f.result() for f in futures]
elapsed = time.monotonic() - started
cpu_after = resource.getrusage(resource.RUSAGE_SELF)
times = sorted(t for row in rows for t in row[0])
errors, instances = collections.Counter(), collections.Counter()
for row in rows:
errors.update(row[1]); instances.update(row[2])
count = len(times)
return {'duration_seconds': elapsed, 'configured_seconds': duration, 'concurrency': concurrency, 'successes': count, 'reads': sum(v[3] for v in rows), 'writes': sum(v[4] for v in rows), 'successful_rps': count / elapsed, 'errors': dict(errors), 'error_fraction': sum(errors.values()) / max(1, count + sum(errors.values())), 'p50_ms': percentile(times, .5), 'p95_ms': percentile(times, .95), 'p99_ms': percentile(times, .99), 'instances': dict(instances), 'generator_cpu_percent': ((cpu_after.ru_utime + cpu_after.ru_stime) - (cpu_before.ru_utime + cpu_before.ru_stime)) / elapsed * 100}
def pool_snapshot():
result = {}
for cid in command(['docker', 'compose', 'ps', '-q', 'api']).split():
command(['docker', 'kill', '--signal=USR1', cid])
logs = subprocess.check_output(['docker', 'logs', '--tail', '10', cid], text=True, stderr=subprocess.STDOUT)
lines = [line for line in logs.splitlines() if 'pool_stats instance=' in line]
assert lines, 'pool diagnostics signal missing'
stats = json.loads(lines[-1].split('stats=', 1)[1])
code = "from pathlib import Path; import json; print(json.dumps(dict(line.split() for line in Path('/sys/fs/cgroup/cpu.stat').read_text().splitlines())))"
stats['cgroup_cpu'] = json.loads(command(['docker', 'exec', cid, 'python', '-c', code]))
result[cid[:12]] = stats
return result
def trial(concurrency, duration):
actor, chats = fixture()
measure(actor, chats, concurrency, 2)
before = pool_snapshot()
samples = []
done = threading.Event()
with concurrent.futures.ThreadPoolExecutor(max_workers=1) as executor:
probe = executor.submit(telemetry, done, samples)
try:
result = measure(actor, chats, concurrency, duration)
finally:
done.set()
probe.result()
result['pool_before'] = before
result['pool_after'] = pool_snapshot()
result['telemetry'] = samples
result['fixture'] = {'chats': 8, 'messages_before_warmup': 800, 'text_code_points': 64, 'history_limit': 50, 'warmup_seconds': 2, 'write_ratio_target': .2}
return result
def configure(replicas, cpu, memory, mode, pool=8, database_cpu=1):
env = dict(os.environ, API_REPLICAS=str(replicas), API_CPUS=str(cpu), API_MEMORY=str(memory) + 'm', DB_ACCESS_MODE=mode, DB_POOL_MAX=str(pool), POSTGRES_CPUS=str(database_cpu))
subprocess.run(['docker', 'compose', 'up', '-d', '--wait', '--no-deps', 'postgres'], env=env, check=True, stdout=subprocess.DEVNULL)
# Independent runs use a distinct new fixture; application replicas are recreated.
subprocess.run(['docker', 'compose', 'up', '-d', '--wait', '--no-deps', '--scale', 'api=' + str(replicas), 'api'], env=env, check=True, stdout=subprocess.DEVNULL)
c = Client(timeout=1)
deadline = time.monotonic() + 30
observed = set()
while time.monotonic() < deadline:
try:
observed.add(c.call('GET', '/health/ready', schema='Health')['headers']['X-Instance-Id'])
if len(observed) >= replicas:
break
except Exception:
pass
time.sleep(.1)
assert len(observed) == replicas
time.sleep(2)
return env
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--duration', type=float, default=8)
parser.add_argument('--repeats', type=int, default=3)
parser.add_argument('--pilot', action='store_true')
parser.add_argument('--resume', action='store_true')
args = parser.parse_args()
output = ROOT / 'evidence/scaling.json'
result = {'runtime': platform.python_version(), 'host': platform.platform(), 'docker': json.loads(command(['docker', 'info', '--format', '{"cpus":{{.NCPU}},"memory":{{.MemTotal}},"os":"{{.OperatingSystem}}"}'])), 'runs': []}
configs = [('fixed', n, 1 / n, 512 // n, 'optimized', 16) for n in (1, 2, 4)]
configs += [('growing', n, .5, 256, 'optimized', 16) for n in (1, 2, 4)]
configs += [('improvement', 2, .5, 256, 'baseline', 16)]
configs += [('saturation', 4, .5, 256, 'optimized', n) for n in (1, 4, 32, 64)]
configs = [(*row, 8) for row in configs]
configs += [('pool_limit', 1, 1, 512, 'optimized', 32, n) for n in (1, 8)]
configs += [('pool_isolated', 1, 1, 512, 'optimized', 16, n) for n in (1,8)]
configs = [(*row, 1) for row in configs]
configs += [('database_limit', n, .5, 256, 'optimized', 16, 8, .1) for n in (1,2,4)]
if args.resume:
result = json.loads(output.read_text())
if args.pilot:
configs = [('pilot', 2, .5, 256, 'optimized', 16, 8, 1)]
try:
for series, replicas, cpu, memory, mode, concurrency, pool, database_cpu in configs:
completed = [v for v in result['runs'] if v['series'] == series and v['replicas'] == replicas and v['concurrency'] == concurrency and v['db_pool_max'] == pool]
if len(completed) >= args.repeats:
continue
configure(replicas, cpu, memory, mode, pool, database_cpu)
for repeat in range(args.repeats):
if any(v['series'] == series and v['replicas'] == replicas and v['concurrency'] == concurrency and v['db_pool_max'] == pool and v['repeat'] == repeat + 1 for v in result['runs']):
continue
row = trial(concurrency, args.duration)
row.update(series=series, replicas=replicas, api_cpu_each=cpu, api_memory_mib_each=memory, mode=mode, repeat=repeat + 1, db_pool_max=pool, postgres_cpu=database_cpu)
result['runs'].append(row)
output.write_text(json.dumps(result, indent=2))
print(json.dumps({k: row[k] for k in ('series', 'replicas', 'mode', 'repeat', 'concurrency', 'successful_rps', 'p95_ms', 'errors', 'generator_cpu_percent')}), flush=True)
finally:
configure(2, 1, 256, 'optimized')
print('Research complete; default two-replica stand restored.')
if __name__ == '__main__':
main()