Files
mtusi2026lab3/student-tests/benchmark.py
T

153 lines
7.2 KiB
Python

#!/usr/bin/env python3
"""Go generator + container CPU/throttling/memory + API pool-wait counters."""
import json
import os
from pathlib import Path
import subprocess
import sys
import threading
import time
import uuid
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / 'tests'))
sys.path.insert(0, str(ROOT / 'scripts'))
from client import Client
from actions import COMPOSE, base_url, command, containers, wait_instances
def sql(query):
return command(COMPOSE + ['exec', '-T', 'postgres', 'psql', '-X', '-v', 'ON_ERROR_STOP=1', '-U', 'messenger', '-d', 'messenger', '-At', '-c', query])
def cpu_snapshot(items):
result = {}
for item in items:
raw = command(['docker', 'exec', item['id'], 'cat', '/sys/fs/cgroup/cpu.stat'])
result[item['id']] = dict((key, int(value)) for key, value in (line.split() for line in raw.splitlines()))
return result
def pool_snapshot(items):
result = {}
for item in items:
if item['service'] != 'api':
continue
text = command(['docker', 'exec', item['id'], 'wget', '-q', '-O', '-', 'http://127.0.0.1:8080/metrics'])
result[item['id']] = {line.split()[0]: float(line.split()[1]) for line in text.splitlines()
if line.startswith(('messenger_db_wait_', 'messenger_db_max_', 'messenger_db_in_use', 'messenger_chat_cache_'))}
return result
def configure(replicas, cpu, memory, pool, db_cpu=1, lb_cpu=.5, cache=False):
environment = dict(os.environ, API_REPLICAS=str(replicas), API_CPUS=str(cpu),
API_MEMORY=memory, DB_MAX_CONNECTIONS=str(pool),
DB_CPUS=str(db_cpu), LB_CPUS=str(lb_cpu), CHAT_CACHE_ENABLED=str(cache).lower())
command(COMPOSE + ['up', '-d', '--no-deps', '--scale', f'api={replicas}', 'api', 'postgres', 'lb'], env=environment)
# Repeated observations must come from running CURRENT containers, not stale DNS.
expected = {x['instance'] for x in containers('api', running=True)}
deadline = time.monotonic() + 90
while time.monotonic() < deadline:
observed = set(wait_instances(replicas))
if observed == expected:
return
time.sleep(1)
raise RuntimeError('LB did not converge to current replica set')
def run_case(label, concurrency=32, duration=20, warmup=3, action=None):
c = Client(base_url())
actor = c.user('bench')
chat = c.call('POST', '/api/v1/chats', 201, actor=actor,
body={'title': 'Benchmark', 'member_ids': [actor['id']]}, schema='Chat')['json']
sql(f"INSERT INTO messages(id,chat_id,sender_id,text,created_at) SELECT md5('{chat['id']}'||n)::uuid,'{chat['id']}','{actor['id']}',repeat('s',200),'2026-01-01T00:00:00Z'::timestamptz+n*interval '1 second' FROM generate_series(1,1000) n")
count_before = int(sql('SELECT count(*) FROM messages'))
items = [x for x in containers(running=True) if x['service'] in ('api', 'lb', 'postgres', 'redis')]
before_cpu = cpu_snapshot(items)
before_pool = pool_snapshot(items)
samples, sampling_errors = [], []
finished = threading.Event()
def sample():
while not finished.is_set():
try:
text = command(['docker', 'stats', '--no-stream', '--format', '{{json .}}'] + [x['id'] for x in items])
samples.append({'elapsed_seconds': round(time.monotonic() - started, 3),
'containers': [json.loads(line) for line in text.splitlines()]})
except RuntimeError as exc:
sampling_errors.append(type(exc).__name__)
finished.wait(.5)
configuration = {'base_url': 'http://lb:8080', 'token': actor['token'], 'chat': chat['id'],
'concurrency': concurrency, 'duration': duration, 'warmup': warmup,
'run_id': uuid.uuid4().hex}
started = time.monotonic()
monitor = threading.Thread(target=sample)
monitor.start()
proc = subprocess.Popen(COMPOSE + ['run', '--rm', '-T', '--no-deps', 'loadgen'], cwd=ROOT,
stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
action_result = {}
def mutate():
try:
time.sleep(warmup + 5)
at = time.monotonic() - started
result = action()
action_result.update({'started_seconds': at, 'completed_seconds': time.monotonic()-started,
'result': result})
except Exception as exc:
action_result['error'] = str(exc)
mutator = threading.Thread(target=mutate) if action else None
if mutator:
mutator.start()
try:
out, err = proc.communicate(json.dumps(configuration).encode(), timeout=duration+warmup+120)
finally:
finished.set()
monitor.join()
if mutator:
mutator.join()
if proc.returncode or 'error' in action_result:
raise RuntimeError('Benchmark failed: '+err.decode(errors='replace')[-700:] + str(action_result.get('error', '')))
result = json.loads(out)
elapsed = time.monotonic() - started
result.update({'label': label, 'workload': 'closed-loop, persistent connections, 80% GET history limit=50 / 20% POST text=200 bytes, no retries',
'warmup_seconds': warmup, 'requested_duration_seconds': duration,
'seed_messages_in_new_chat': 1000, 'total_messages_before': count_before,
'total_messages_after': int(sql('SELECT count(*) FROM messages')),
'resources': items, 'resource_window_seconds': elapsed,
'resource_window_includes': 'generator container startup, warmup and measurement',
'resource_samples': samples, 'sampling_errors': sampling_errors,
'action': action_result})
after_items = {x['id']: x for x in containers(running=True)}
survived = [x for x in items if x['id'] in after_items]
after_cpu = cpu_snapshot(survived)
after_pool = pool_snapshot(survived)
result['cgroup_deltas'] = {}
for item in survived:
ident = item['id']
diff = {k: after_cpu[ident][k] - before_cpu[ident][k] for k in before_cpu[ident]}
quota = item['nano_cpus'] / 1e9
diff['quota_utilization_ratio'] = diff['usage_usec'] / 1e6 / elapsed / quota
diff['name'] = item['name']
result['cgroup_deltas'][ident] = diff
result['pool_deltas'] = {ident: {key: after_pool[ident][key] - before_pool[ident][key]
for key in before_pool[ident] if key.startswith(('messenger_db_wait_', 'messenger_chat_cache_'))}
for ident in after_pool}
path = ROOT / 'evidence' / 'runs' / (label + '.json')
path.parent.mkdir(exist_ok=True)
path.write_text(json.dumps(result, indent=2) + '\n', encoding='utf-8')
print(f"{label}: {result['successful_rps']:.1f} RPS, errors={result['failed_requests']}, p95={result['all_attempt_latency_ms']['p95']:.1f} ms", flush=True)
return result
if __name__ == '__main__':
import argparse
parser = argparse.ArgumentParser()
parser.add_argument('--label', default='quick')
parser.add_argument('--duration', type=float, default=20)
parser.add_argument('--concurrency', type=int, default=32)
args = parser.parse_args()
run_case(args.label, args.concurrency, args.duration)