154 lines
5.1 KiB
Python
154 lines
5.1 KiB
Python
#!/usr/bin/env python3
|
|
import os
|
|
import json
|
|
import urllib.request
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
from statistics import median
|
|
import subprocess
|
|
import time
|
|
|
|
from client import Client, check
|
|
|
|
|
|
ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
|
COMPOSE = os.environ.get('COMPOSE_BIN', 'docker-compose')
|
|
BASE_URL = os.environ.get('BASE_URL', 'http://localhost:8080')
|
|
USERS = 4
|
|
CHATS_PER_USER = 15
|
|
REQUESTS_PER_USER = 250
|
|
WORKERS = 8
|
|
|
|
for proxy_name in ('HTTP_PROXY', 'HTTPS_PROXY', 'ALL_PROXY', 'http_proxy', 'https_proxy', 'all_proxy'):
|
|
os.environ.pop(proxy_name, None)
|
|
os.environ['NO_PROXY'] = 'localhost,127.0.0.1,::1'
|
|
os.environ['no_proxy'] = os.environ['NO_PROXY']
|
|
|
|
|
|
def restart_api(cache_enabled):
|
|
env = dict(os.environ)
|
|
env['REDIS_ENABLED'] = 'true'
|
|
env['CACHE_ENABLED'] = 'true' if cache_enabled else 'false'
|
|
subprocess.run(
|
|
[COMPOSE, '-f', 'docker-compose.local.yaml', 'up', '-d', '--force-recreate', 'api'],
|
|
cwd=ROOT,
|
|
env=env,
|
|
check=True,
|
|
stdout=subprocess.DEVNULL,
|
|
stderr=subprocess.DEVNULL,
|
|
)
|
|
|
|
deadline = time.monotonic() + 90
|
|
while time.monotonic() < deadline:
|
|
try:
|
|
Client(BASE_URL, timeout=2).call('GET', '/health/ready', schema='Health')
|
|
return
|
|
except Exception:
|
|
time.sleep(0.5)
|
|
raise RuntimeError('API did not become ready after restart')
|
|
|
|
|
|
def percentile(values, percent):
|
|
values = sorted(values)
|
|
index = max(0, (len(values) * percent + 99) // 100 - 1)
|
|
return values[index]
|
|
|
|
|
|
def benchmark(actors):
|
|
requests = [(actor, i) for i in range(REQUESTS_PER_USER) for actor in actors]
|
|
latencies = []
|
|
failures = []
|
|
started = time.monotonic()
|
|
|
|
def request(item):
|
|
actor, _ = item
|
|
request_started = time.monotonic()
|
|
request = urllib.request.Request(
|
|
BASE_URL + '/api/v1/chats?limit=50',
|
|
headers={
|
|
'Accept': 'application/json',
|
|
'Authorization': 'Bearer ' + actor['token'],
|
|
},
|
|
)
|
|
try:
|
|
with urllib.request.urlopen(request, timeout=10) as response:
|
|
data = response.read()
|
|
status = response.status
|
|
check(status == 200,
|
|
'unexpected HTTP status: ' + str(status) + ' body=' + data.decode())
|
|
result = json.loads(data)
|
|
check(len(result['items']) == 30, 'unexpected chat count')
|
|
check(all(actor['id'] in chat['member_ids'] for chat in result['items']),
|
|
'response contains another user chat')
|
|
return time.monotonic() - request_started, None
|
|
except Exception as exc:
|
|
return 0, type(exc).__name__ + ': ' + str(exc)
|
|
|
|
with ThreadPoolExecutor(max_workers=WORKERS) as pool:
|
|
for latency, error in pool.map(request, requests):
|
|
if error is None:
|
|
latencies.append(latency)
|
|
else:
|
|
failures.append(error)
|
|
|
|
duration = time.monotonic() - started
|
|
if failures:
|
|
raise RuntimeError(
|
|
'API benchmark failed: %d/%d requests; first error: %s'
|
|
% (len(failures), len(requests), failures[0])
|
|
)
|
|
return {
|
|
'requests': len(latencies),
|
|
'workers': WORKERS,
|
|
'duration_seconds': duration,
|
|
'rps': len(latencies) / duration,
|
|
'p50_ms': median(latencies) * 1000,
|
|
'p95_ms': percentile(latencies, 95) * 1000,
|
|
'p99_ms': percentile(latencies, 99) * 1000,
|
|
}
|
|
|
|
|
|
def main():
|
|
client = Client(BASE_URL, timeout=10)
|
|
actors = [client.user('perf_api_' + str(i)) for i in range(USERS)]
|
|
|
|
for index, owner in enumerate(actors):
|
|
member = actors[(index + 1) % len(actors)]
|
|
for chat_index in range(CHATS_PER_USER):
|
|
client.call(
|
|
'POST',
|
|
'/api/v1/chats',
|
|
expected=201,
|
|
actor=owner,
|
|
body={
|
|
'title': 'API performance chat ' + str(index) + '-' + str(chat_index),
|
|
'member_ids': [owner['id'], member['id']],
|
|
},
|
|
schema='Chat',
|
|
)
|
|
|
|
results = []
|
|
try:
|
|
for trial in range(2):
|
|
for enabled in (False, True):
|
|
restart_api(enabled)
|
|
for actor in actors:
|
|
for _ in range(5):
|
|
client.call('GET', '/api/v1/chats?limit=50', actor=actor, schema='ChatPage')
|
|
print(
|
|
'running trial %d, cache=%s, requests=%d' %
|
|
(trial + 1, enabled, len(actors) * REQUESTS_PER_USER),
|
|
flush=True,
|
|
)
|
|
result = benchmark(actors)
|
|
result.update(trial=trial + 1, cache_enabled=enabled)
|
|
results.append(result)
|
|
print(json.dumps(result), flush=True)
|
|
finally:
|
|
restart_api(True)
|
|
print(json.dumps({'results': results, 'users': USERS, 'chats': USERS * CHATS_PER_USER,
|
|
'sessions_cached_in_both_modes': True}), flush=True)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|