143 lines
6.9 KiB
Python
143 lines
6.9 KiB
Python
"""Closed-loop POST benchmark for the isolated track1 database, through nginx only."""
|
|
import argparse
|
|
import csv
|
|
from collections import Counter
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
from datetime import datetime, timezone
|
|
import json
|
|
import math
|
|
import os
|
|
from pathlib import Path
|
|
import sys
|
|
import threading
|
|
import time
|
|
from uuid import uuid4
|
|
|
|
import psycopg
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests'))
|
|
from client import Client, Failure
|
|
|
|
|
|
def db():
|
|
return psycopg.connect(host=os.environ['POSTGRES_HOST'], dbname=os.environ['POSTGRES_DB'],
|
|
user=os.environ['POSTGRES_USER'], password=os.environ['POSTGRES_PASSWORD'],
|
|
autocommit=True, application_name='track1-monitor', connect_timeout=3)
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument('--name', required=True)
|
|
parser.add_argument('--concurrency', type=int, default=32)
|
|
parser.add_argument('--duration', type=float, default=15)
|
|
parser.add_argument('--warmup', type=float, default=3)
|
|
args = parser.parse_args()
|
|
assert 1 <= args.concurrency <= 128 and 1 <= args.duration <= 120
|
|
assert os.environ.get('TRACK1_BENCHMARK') == '1'
|
|
assert Path(args.name).name == args.name
|
|
# Never reset the normal lab database. This project has separate synthetic volumes.
|
|
with db() as conn:
|
|
assert conn.execute('SELECT current_database()').fetchone()[0] == 'track1'
|
|
conn.execute('TRUNCATE message_idempotency, messages, chat_members, chats, users')
|
|
client = Client('http://nginx:8080', timeout=5)
|
|
actor = client.user('scale')
|
|
chat = client.call('POST', '/api/v1/chats', 201, actor=actor,
|
|
body={'title': 'Scaling', 'member_ids': [actor['id']]}, schema='Chat')['json']
|
|
payload = 'm' * 64
|
|
for _ in range(100):
|
|
client.message(actor, chat, payload)
|
|
stop_monitor = threading.Event()
|
|
waits = []
|
|
started = None
|
|
started_at = None
|
|
|
|
def start_clock():
|
|
nonlocal started, started_at
|
|
started = time.monotonic()
|
|
started_at = datetime.now(timezone.utc).isoformat()
|
|
|
|
gate = threading.Barrier(args.concurrency + 1, action=start_clock)
|
|
|
|
def monitor():
|
|
with db() as conn:
|
|
while not stop_monitor.wait(0.5):
|
|
rows = conn.execute("""
|
|
SELECT state, coalesce(wait_event_type, 'CPU'), coalesce(wait_event, ''), count(*)
|
|
FROM pg_stat_activity WHERE datname=current_database()
|
|
AND application_name <> 'track1-monitor' AND backend_type='client backend'
|
|
GROUP BY state, wait_event_type, wait_event
|
|
""").fetchall()
|
|
waits.append({'second': round(time.monotonic() - started, 3),
|
|
'states': [{'state': s, 'wait_type': t, 'wait_event': e, 'connections': n}
|
|
for s, t, e, n in rows]})
|
|
|
|
def worker(_):
|
|
local = Client('http://nginx:8080', timeout=5)
|
|
gate.wait(timeout=120)
|
|
events, confirmations, instances = [], set(), Counter()
|
|
attempts = 0
|
|
while time.monotonic() - started < args.warmup + args.duration:
|
|
begin = time.monotonic()
|
|
attempts += 1
|
|
try:
|
|
response = local.message(actor, chat, payload, key=uuid4().hex)
|
|
confirmations.add(response['json']['id'])
|
|
instances[response['headers']['X-Instance-Id']] += 1
|
|
error = None
|
|
except Failure as exc:
|
|
error = str(exc) # Client deliberately omits credentials and bodies.
|
|
end = time.monotonic()
|
|
if begin - started >= args.warmup:
|
|
events.append((begin-started-args.warmup, end-begin, error))
|
|
return events, confirmations, instances, attempts
|
|
|
|
with ThreadPoolExecutor(max_workers=args.concurrency) as pool:
|
|
futures = [pool.submit(worker, i) for i in range(args.concurrency)]
|
|
gate.wait(timeout=120)
|
|
monitor_thread = threading.Thread(target=monitor)
|
|
monitor_thread.start()
|
|
results = [future.result() for future in futures]
|
|
elapsed = time.monotonic() - started
|
|
stop_monitor.set()
|
|
monitor_thread.join()
|
|
events = [event for result in results for event in result[0]]
|
|
successful = sorted(duration for _, duration, error in events if error is None)
|
|
errors = Counter(error for _, _, error in events if error is not None)
|
|
confirmed = set().union(*(result[1] for result in results))
|
|
instances = sum((result[2] for result in results), Counter())
|
|
with db() as conn:
|
|
stored = {str(row[0]) for row in conn.execute('SELECT id FROM messages WHERE chat_id=%s', (chat['id'],))}
|
|
keys = conn.execute('SELECT count(*) FROM message_idempotency WHERE chat_id=%s', (chat['id'],)).fetchone()[0]
|
|
assert confirmed <= stored, 'Confirmed message lost'
|
|
assert len(stored) == keys, 'Message/idempotency count mismatch'
|
|
assert len(stored) <= 100 + sum(result[3] for result in results), 'Duplicate messages'
|
|
if len(confirmed) == sum(result[3] for result in results):
|
|
assert len(stored) == 100 + len(confirmed)
|
|
seconds = []
|
|
for second in range(math.ceil(args.duration)):
|
|
subset = [e for e in events if second <= e[0] < second+1]
|
|
seconds.append({'second': second, 'successes': sum(e[2] is None for e in subset),
|
|
'errors': sum(e[2] is not None for e in subset)})
|
|
measured = elapsed - args.warmup
|
|
def percentile(p):
|
|
return round(successful[max(0, math.ceil(len(successful)*p)-1)]*1000, 3) if successful else None
|
|
report = {'name': args.name, 'concurrency': args.concurrency, 'warmup_seconds': args.warmup,
|
|
'workload_started_at': started_at,
|
|
'requested_seconds': args.duration, 'elapsed_measurement_seconds': round(measured, 3),
|
|
'successes': len(successful), 'errors': dict(errors), 'failed_requests': sum(errors.values()),
|
|
'successful_rps': round(len(successful)/measured, 3),
|
|
'latency_ms': {'p50': percentile(.5), 'p95': percentile(.95), 'p99': percentile(.99)},
|
|
'instances': dict(instances), 'confirmed_messages_preserved': True,
|
|
'message_count': len(stored), 'seconds': seconds, 'db_samples': waits,
|
|
'note': 'Closed loop; 64-byte text; 100 initial messages; one chat; no retries; success-only latency; drain included.'}
|
|
Path('/results', args.name+'.json').write_text(json.dumps(report, indent=2)+'\n')
|
|
with Path('/results', args.name+'-requests.csv').open('w', newline='') as stream:
|
|
writer = csv.writer(stream)
|
|
writer.writerow(['start_seconds', 'latency_ms', 'error'])
|
|
writer.writerows((round(s, 6), round(d*1000, 3), e or '') for s, d, e in sorted(events))
|
|
print(json.dumps({k: report[k] for k in ('name','successful_rps','failed_requests','latency_ms','instances')}), flush=True)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|