Files
mtusi2026lab3/scripts/benchmark.py

98 lines
4.6 KiB
Python

#!/usr/bin/env python3
"""Persistent-connection async POST benchmark with raw per-request samples.
Closed loop, no retries; successful RPS counts HTTP acknowledgements, not commits.
Bootstrap is outside measured time; credentials are never written to disk.
"""
import argparse
import asyncio
from collections import Counter
import json
import math
import os
from pathlib import Path
import sys
import time
import uuid
import httpx
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests'))
from client import Client
def percentile(values, p):
values = sorted(values)
return round(values[max(0, math.ceil(len(values) * p) - 1)], 3) if values else None
async def measure(base, actor, chat, duration, warmup, concurrency):
samples = []
limits = httpx.Limits(max_connections=concurrency, max_keepalive_connections=concurrency)
async with httpx.AsyncClient(base_url=base, timeout=5, limits=limits, trust_env=False) as client:
start = time.monotonic()
measured_start, deadline = start + warmup, start + warmup + duration
async def worker(worker_id):
index = 0
while time.monotonic() < deadline:
before = time.monotonic()
try:
response = await client.post('/api/v1/chats/' + chat['id'] + '/messages',
headers={'Authorization': 'Bearer ' + actor['token'], 'Idempotency-Key': uuid.uuid4().hex},
json={'text': 'load-%04d-%08d' % (worker_id, index)})
status = response.status_code
instance = response.headers.get('x-instance-id', '')
except httpx.HTTPError:
status, instance = 0, ''
after = time.monotonic()
if before >= measured_start:
samples.append({'started_s': round(before - measured_start, 6),
'latency_ms': round((after - before) * 1000, 3),
'status': status, 'instance': instance})
index += 1
await asyncio.gather(*(worker(i) for i in range(concurrency)))
elapsed = time.monotonic() - measured_start
return samples, elapsed
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--base-url', default=os.getenv('BASE_URL', 'http://localhost:8080'))
parser.add_argument('--duration', type=float, default=20)
parser.add_argument('--warmup', type=float, default=5)
parser.add_argument('--concurrency', type=int, default=32)
parser.add_argument('--output', required=True)
args = parser.parse_args()
if not 1 <= args.concurrency <= 256 or not 1 <= args.duration <= 3600 or args.warmup < 0:
parser.error('invalid load parameters')
alice, _, _, chat = Client(args.base_url).bootstrap()
samples, elapsed = asyncio.run(measure(args.base_url, alice, chat, args.duration, args.warmup, args.concurrency))
success = [s['latency_ms'] for s in samples if s['status'] == 201]
report = {'workload': 'closed-loop POST messages; unique keys, persistent connections, no retries',
'duration_seconds': args.duration, 'warmup_seconds': args.warmup,
'elapsed_seconds': round(elapsed, 3), 'concurrency': args.concurrency,
'successful_requests': len(success), 'failed_requests': len(samples) - len(success),
'successful_rps': round(len(success) / elapsed, 3),
'error_ratio': round((len(samples) - len(success)) / len(samples), 6) if samples else None,
'successful_latency_ms': {p: percentile(success, v) for p, v in [('p50', .5), ('p95', .95), ('p99', .99)]},
'status_counts': dict(Counter(s['status'] for s in samples)),
'instance_counts': dict(Counter(s['instance'] for s in samples)),
'dataset': {'initial_messages_in_new_chat': 0, 'measured_attempts': len(samples),
'text_template_codepoints': 18},
'limits': 'Success-only latency; closed-loop coordinated omission. Client/host can bottleneck. '
'Warmup inserts remain in history; requests started in warmup excluded from samples.'}
dest = Path(args.output)
dest.parent.mkdir(parents=True, exist_ok=True)
dest.write_text(json.dumps(report, indent=2) + '\n', encoding='utf-8')
raw = dest.with_suffix('.jsonl')
raw.write_text(''.join(json.dumps(s) + '\n' for s in samples), encoding='utf-8')
print(json.dumps(report, indent=2))
if not success:
raise SystemExit('No successful writes')
if __name__ == '__main__':
main()