"""Closed-loop POST benchmark for the isolated track1 database, through nginx only.""" import argparse import csv from collections import Counter from concurrent.futures import ThreadPoolExecutor from datetime import datetime, timezone import json import math import os from pathlib import Path import sys import threading import time from uuid import uuid4 import psycopg sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests')) from client import Client, Failure def db(): return psycopg.connect(host=os.environ['POSTGRES_HOST'], dbname=os.environ['POSTGRES_DB'], user=os.environ['POSTGRES_USER'], password=os.environ['POSTGRES_PASSWORD'], autocommit=True, application_name='track1-monitor', connect_timeout=3) def main(): parser = argparse.ArgumentParser() parser.add_argument('--name', required=True) parser.add_argument('--concurrency', type=int, default=32) parser.add_argument('--duration', type=float, default=15) parser.add_argument('--warmup', type=float, default=3) args = parser.parse_args() assert 1 <= args.concurrency <= 128 and 1 <= args.duration <= 120 assert os.environ.get('TRACK1_BENCHMARK') == '1' assert Path(args.name).name == args.name # Never reset the normal lab database. This project has separate synthetic volumes. with db() as conn: assert conn.execute('SELECT current_database()').fetchone()[0] == 'track1' conn.execute('TRUNCATE message_idempotency, messages, chat_members, chats, users') client = Client('http://nginx:8080', timeout=5) actor = client.user('scale') chat = client.call('POST', '/api/v1/chats', 201, actor=actor, body={'title': 'Scaling', 'member_ids': [actor['id']]}, schema='Chat')['json'] payload = 'm' * 64 for _ in range(100): client.message(actor, chat, payload) stop_monitor = threading.Event() waits = [] started = None started_at = None def start_clock(): nonlocal started, started_at started = time.monotonic() started_at = datetime.now(timezone.utc).isoformat() gate = threading.Barrier(args.concurrency + 1, action=start_clock) def monitor(): with db() as conn: while not stop_monitor.wait(0.5): rows = conn.execute(""" SELECT state, coalesce(wait_event_type, 'CPU'), coalesce(wait_event, ''), count(*) FROM pg_stat_activity WHERE datname=current_database() AND application_name <> 'track1-monitor' AND backend_type='client backend' GROUP BY state, wait_event_type, wait_event """).fetchall() waits.append({'second': round(time.monotonic() - started, 3), 'states': [{'state': s, 'wait_type': t, 'wait_event': e, 'connections': n} for s, t, e, n in rows]}) def worker(_): local = Client('http://nginx:8080', timeout=5) gate.wait(timeout=120) events, confirmations, instances = [], set(), Counter() attempts = 0 while time.monotonic() - started < args.warmup + args.duration: begin = time.monotonic() attempts += 1 try: response = local.message(actor, chat, payload, key=uuid4().hex) confirmations.add(response['json']['id']) instances[response['headers']['X-Instance-Id']] += 1 error = None except Failure as exc: error = str(exc) # Client deliberately omits credentials and bodies. end = time.monotonic() if begin - started >= args.warmup: events.append((begin-started-args.warmup, end-begin, error)) return events, confirmations, instances, attempts with ThreadPoolExecutor(max_workers=args.concurrency) as pool: futures = [pool.submit(worker, i) for i in range(args.concurrency)] gate.wait(timeout=120) monitor_thread = threading.Thread(target=monitor) monitor_thread.start() results = [future.result() for future in futures] elapsed = time.monotonic() - started stop_monitor.set() monitor_thread.join() events = [event for result in results for event in result[0]] successful = sorted(duration for _, duration, error in events if error is None) errors = Counter(error for _, _, error in events if error is not None) confirmed = set().union(*(result[1] for result in results)) instances = sum((result[2] for result in results), Counter()) with db() as conn: stored = {str(row[0]) for row in conn.execute('SELECT id FROM messages WHERE chat_id=%s', (chat['id'],))} keys = conn.execute('SELECT count(*) FROM message_idempotency WHERE chat_id=%s', (chat['id'],)).fetchone()[0] assert confirmed <= stored, 'Confirmed message lost' assert len(stored) == keys, 'Message/idempotency count mismatch' assert len(stored) <= 100 + sum(result[3] for result in results), 'Duplicate messages' if len(confirmed) == sum(result[3] for result in results): assert len(stored) == 100 + len(confirmed) seconds = [] for second in range(math.ceil(args.duration)): subset = [e for e in events if second <= e[0] < second+1] seconds.append({'second': second, 'successes': sum(e[2] is None for e in subset), 'errors': sum(e[2] is not None for e in subset)}) measured = elapsed - args.warmup def percentile(p): return round(successful[max(0, math.ceil(len(successful)*p)-1)]*1000, 3) if successful else None report = {'name': args.name, 'concurrency': args.concurrency, 'warmup_seconds': args.warmup, 'workload_started_at': started_at, 'requested_seconds': args.duration, 'elapsed_measurement_seconds': round(measured, 3), 'successes': len(successful), 'errors': dict(errors), 'failed_requests': sum(errors.values()), 'successful_rps': round(len(successful)/measured, 3), 'latency_ms': {'p50': percentile(.5), 'p95': percentile(.95), 'p99': percentile(.99)}, 'instances': dict(instances), 'confirmed_messages_preserved': True, 'message_count': len(stored), 'seconds': seconds, 'db_samples': waits, 'note': 'Closed loop; 64-byte text; 100 initial messages; one chat; no retries; success-only latency; drain included.'} Path('/results', args.name+'.json').write_text(json.dumps(report, indent=2)+'\n') with Path('/results', args.name+'-requests.csv').open('w', newline='') as stream: writer = csv.writer(stream) writer.writerow(['start_seconds', 'latency_ms', 'error']) writer.writerows((round(s, 6), round(d*1000, 3), e or '') for s, d, e in sorted(events)) print(json.dumps({k: report[k] for k in ('name','successful_rps','failed_requests','latency_ms','instances')}), flush=True) if __name__ == '__main__': main()