forked from mtusi2026/mtusi2026lab3
153 lines
7.2 KiB
Python
153 lines
7.2 KiB
Python
#!/usr/bin/env python3
|
|
"""Go generator + container CPU/throttling/memory + API pool-wait counters."""
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import subprocess
|
|
import sys
|
|
import threading
|
|
import time
|
|
import uuid
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT / 'tests'))
|
|
sys.path.insert(0, str(ROOT / 'scripts'))
|
|
from client import Client
|
|
from actions import COMPOSE, base_url, command, containers, wait_instances
|
|
|
|
|
|
def sql(query):
|
|
return command(COMPOSE + ['exec', '-T', 'postgres', 'psql', '-X', '-v', 'ON_ERROR_STOP=1', '-U', 'messenger', '-d', 'messenger', '-At', '-c', query])
|
|
|
|
|
|
def cpu_snapshot(items):
|
|
result = {}
|
|
for item in items:
|
|
raw = command(['docker', 'exec', item['id'], 'cat', '/sys/fs/cgroup/cpu.stat'])
|
|
result[item['id']] = dict((key, int(value)) for key, value in (line.split() for line in raw.splitlines()))
|
|
return result
|
|
|
|
|
|
def pool_snapshot(items):
|
|
result = {}
|
|
for item in items:
|
|
if item['service'] != 'api':
|
|
continue
|
|
text = command(['docker', 'exec', item['id'], 'wget', '-q', '-O', '-', 'http://127.0.0.1:8080/metrics'])
|
|
result[item['id']] = {line.split()[0]: float(line.split()[1]) for line in text.splitlines()
|
|
if line.startswith(('messenger_db_wait_', 'messenger_db_max_', 'messenger_db_in_use', 'messenger_chat_cache_'))}
|
|
return result
|
|
|
|
|
|
def configure(replicas, cpu, memory, pool, db_cpu=1, lb_cpu=.5, cache=False):
|
|
environment = dict(os.environ, API_REPLICAS=str(replicas), API_CPUS=str(cpu),
|
|
API_MEMORY=memory, DB_MAX_CONNECTIONS=str(pool),
|
|
DB_CPUS=str(db_cpu), LB_CPUS=str(lb_cpu), CHAT_CACHE_ENABLED=str(cache).lower())
|
|
command(COMPOSE + ['up', '-d', '--no-deps', '--scale', f'api={replicas}', 'api', 'postgres', 'lb'], env=environment)
|
|
# Repeated observations must come from running CURRENT containers, not stale DNS.
|
|
expected = {x['instance'] for x in containers('api', running=True)}
|
|
deadline = time.monotonic() + 90
|
|
while time.monotonic() < deadline:
|
|
observed = set(wait_instances(replicas))
|
|
if observed == expected:
|
|
return
|
|
time.sleep(1)
|
|
raise RuntimeError('LB did not converge to current replica set')
|
|
|
|
|
|
def run_case(label, concurrency=32, duration=20, warmup=3, action=None):
|
|
c = Client(base_url())
|
|
actor = c.user('bench')
|
|
chat = c.call('POST', '/api/v1/chats', 201, actor=actor,
|
|
body={'title': 'Benchmark', 'member_ids': [actor['id']]}, schema='Chat')['json']
|
|
sql(f"INSERT INTO messages(id,chat_id,sender_id,text,created_at) SELECT md5('{chat['id']}'||n)::uuid,'{chat['id']}','{actor['id']}',repeat('s',200),'2026-01-01T00:00:00Z'::timestamptz+n*interval '1 second' FROM generate_series(1,1000) n")
|
|
count_before = int(sql('SELECT count(*) FROM messages'))
|
|
items = [x for x in containers(running=True) if x['service'] in ('api', 'lb', 'postgres', 'redis')]
|
|
before_cpu = cpu_snapshot(items)
|
|
before_pool = pool_snapshot(items)
|
|
samples, sampling_errors = [], []
|
|
finished = threading.Event()
|
|
|
|
def sample():
|
|
while not finished.is_set():
|
|
try:
|
|
text = command(['docker', 'stats', '--no-stream', '--format', '{{json .}}'] + [x['id'] for x in items])
|
|
samples.append({'elapsed_seconds': round(time.monotonic() - started, 3),
|
|
'containers': [json.loads(line) for line in text.splitlines()]})
|
|
except RuntimeError as exc:
|
|
sampling_errors.append(type(exc).__name__)
|
|
finished.wait(.5)
|
|
|
|
configuration = {'base_url': 'http://lb:8080', 'token': actor['token'], 'chat': chat['id'],
|
|
'concurrency': concurrency, 'duration': duration, 'warmup': warmup,
|
|
'run_id': uuid.uuid4().hex}
|
|
started = time.monotonic()
|
|
monitor = threading.Thread(target=sample)
|
|
monitor.start()
|
|
proc = subprocess.Popen(COMPOSE + ['run', '--rm', '-T', '--no-deps', 'loadgen'], cwd=ROOT,
|
|
stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
|
action_result = {}
|
|
|
|
def mutate():
|
|
try:
|
|
time.sleep(warmup + 5)
|
|
at = time.monotonic() - started
|
|
result = action()
|
|
action_result.update({'started_seconds': at, 'completed_seconds': time.monotonic()-started,
|
|
'result': result})
|
|
except Exception as exc:
|
|
action_result['error'] = str(exc)
|
|
|
|
mutator = threading.Thread(target=mutate) if action else None
|
|
if mutator:
|
|
mutator.start()
|
|
try:
|
|
out, err = proc.communicate(json.dumps(configuration).encode(), timeout=duration+warmup+120)
|
|
finally:
|
|
finished.set()
|
|
monitor.join()
|
|
if mutator:
|
|
mutator.join()
|
|
if proc.returncode or 'error' in action_result:
|
|
raise RuntimeError('Benchmark failed: '+err.decode(errors='replace')[-700:] + str(action_result.get('error', '')))
|
|
result = json.loads(out)
|
|
elapsed = time.monotonic() - started
|
|
result.update({'label': label, 'workload': 'closed-loop, persistent connections, 80% GET history limit=50 / 20% POST text=200 bytes, no retries',
|
|
'warmup_seconds': warmup, 'requested_duration_seconds': duration,
|
|
'seed_messages_in_new_chat': 1000, 'total_messages_before': count_before,
|
|
'total_messages_after': int(sql('SELECT count(*) FROM messages')),
|
|
'resources': items, 'resource_window_seconds': elapsed,
|
|
'resource_window_includes': 'generator container startup, warmup and measurement',
|
|
'resource_samples': samples, 'sampling_errors': sampling_errors,
|
|
'action': action_result})
|
|
after_items = {x['id']: x for x in containers(running=True)}
|
|
survived = [x for x in items if x['id'] in after_items]
|
|
after_cpu = cpu_snapshot(survived)
|
|
after_pool = pool_snapshot(survived)
|
|
result['cgroup_deltas'] = {}
|
|
for item in survived:
|
|
ident = item['id']
|
|
diff = {k: after_cpu[ident][k] - before_cpu[ident][k] for k in before_cpu[ident]}
|
|
quota = item['nano_cpus'] / 1e9
|
|
diff['quota_utilization_ratio'] = diff['usage_usec'] / 1e6 / elapsed / quota
|
|
diff['name'] = item['name']
|
|
result['cgroup_deltas'][ident] = diff
|
|
result['pool_deltas'] = {ident: {key: after_pool[ident][key] - before_pool[ident][key]
|
|
for key in before_pool[ident] if key.startswith(('messenger_db_wait_', 'messenger_chat_cache_'))}
|
|
for ident in after_pool}
|
|
path = ROOT / 'evidence' / 'runs' / (label + '.json')
|
|
path.parent.mkdir(exist_ok=True)
|
|
path.write_text(json.dumps(result, indent=2) + '\n', encoding='utf-8')
|
|
print(f"{label}: {result['successful_rps']:.1f} RPS, errors={result['failed_requests']}, p95={result['all_attempt_latency_ms']['p95']:.1f} ms", flush=True)
|
|
return result
|
|
|
|
|
|
if __name__ == '__main__':
|
|
import argparse
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument('--label', default='quick')
|
|
parser.add_argument('--duration', type=float, default=20)
|
|
parser.add_argument('--concurrency', type=int, default=32)
|
|
args = parser.parse_args()
|
|
run_case(args.label, args.concurrency, args.duration)
|