117 lines
3.8 KiB
Python
117 lines
3.8 KiB
Python
#!/usr/bin/env python3
|
|
"""Diagnostic closed-loop load test. Does not modify public tests."""
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
from collections import Counter
|
|
from pathlib import Path
|
|
import json
|
|
import math
|
|
import re
|
|
import sys
|
|
import threading
|
|
import time
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT / "tests"))
|
|
|
|
from client import Client, Failure, LAB, arguments, check, run
|
|
|
|
|
|
def main():
|
|
def configure(p):
|
|
p.add_argument('--duration', type=float, default=90)
|
|
p.add_argument('--concurrency', type=int, default=8)
|
|
p.add_argument('--output', default='evidence/rolling-update-diagnostic.json')
|
|
|
|
args = arguments('Diagnostic synthetic POST message load', configure)
|
|
check(1 <= args.duration <= 3600, 'duration must be 1..3600 seconds')
|
|
check(1 <= args.concurrency <= 128, 'concurrency must be 1..128')
|
|
|
|
c = Client(args.base_url, args.ca_file)
|
|
alice, bob, eve, chat = c.bootstrap()
|
|
barrier = threading.Barrier(args.concurrency)
|
|
|
|
def worker(worker_id):
|
|
local = Client(args.base_url, args.ca_file, timeout=5)
|
|
barrier.wait()
|
|
deadline = time.monotonic() + args.duration
|
|
latencies = []
|
|
errors = Counter()
|
|
examples = []
|
|
index = 0
|
|
|
|
while time.monotonic() < deadline:
|
|
start = time.monotonic()
|
|
try:
|
|
local.message(alice, chat, 'diagnostic-%s-%s' % (worker_id, index))
|
|
latencies.append(time.monotonic() - start)
|
|
except Failure as exc:
|
|
message = str(exc)
|
|
match = re.search(r'received (\d{3})', message)
|
|
|
|
if match:
|
|
category = 'http_' + match.group(1)
|
|
elif 'connection/TLS failure' in message:
|
|
kind = re.search(r'\(([^)]+)\)', message)
|
|
category = 'connection_' + (kind.group(1) if kind else 'unknown')
|
|
else:
|
|
category = 'other_failure'
|
|
|
|
errors[category] += 1
|
|
if len(examples) < 10:
|
|
examples.append(message[:300])
|
|
|
|
index += 1
|
|
|
|
return latencies, errors, examples
|
|
|
|
started = time.monotonic()
|
|
with ThreadPoolExecutor(max_workers=args.concurrency) as pool:
|
|
results = list(pool.map(worker, range(args.concurrency)))
|
|
elapsed = time.monotonic() - started
|
|
|
|
latencies = sorted(x for values, _, _ in results for x in values)
|
|
errors = Counter()
|
|
examples = []
|
|
|
|
for _, counts, samples in results:
|
|
errors.update(counts)
|
|
examples.extend(samples)
|
|
|
|
total_errors = sum(errors.values())
|
|
total = len(latencies) + total_errors
|
|
|
|
def percentile(p):
|
|
if not latencies:
|
|
return None
|
|
return round(1000 * latencies[max(0, math.ceil(p * len(latencies)) - 1)], 3)
|
|
|
|
report = {
|
|
'lab': LAB,
|
|
'workload': 'closed-loop POST messages; no automatic retries',
|
|
'duration_seconds': round(elapsed, 3),
|
|
'concurrency': args.concurrency,
|
|
'successful_requests': len(latencies),
|
|
'failed_requests': total_errors,
|
|
'error_ratio': round(total_errors / total, 6) if total else None,
|
|
'errors_by_type': dict(errors),
|
|
'successful_rps': round(len(latencies) / elapsed, 3),
|
|
'successful_latency_ms': {
|
|
'p50': percentile(.50),
|
|
'p95': percentile(.95),
|
|
'p99': percentile(.99)
|
|
},
|
|
'error_examples': examples[:20],
|
|
'note': 'Latency percentiles cover successful requests only.'
|
|
}
|
|
|
|
text = json.dumps(report, ensure_ascii=False, indent=2)
|
|
dest = Path(args.output)
|
|
dest.parent.mkdir(parents=True, exist_ok=True)
|
|
dest.write_text(text + '\n', encoding='utf-8')
|
|
print(text)
|
|
check(bool(latencies), 'no successful writes')
|
|
|
|
|
|
if __name__ == '__main__':
|
|
run(main)
|