#!/usr/bin/env python3 """Run closed-loop writes while API replicas are replaced one at a time.""" from concurrent.futures import ThreadPoolExecutor import json import math import os from pathlib import Path import subprocess import sys import threading import time ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT / 'tests')) from client import Client, Failure, arguments, check, run def main(): def configure(parser): parser.add_argument('--action-script', required=True) parser.add_argument('--duration', type=float, default=180) parser.add_argument('--concurrency', type=int, default=4) parser.add_argument('--output', default=None) args = arguments('Rolling replacement under write load', configure) check(30 <= args.duration <= 300, 'duration must be 30..300 seconds') check(1 <= args.concurrency <= 32, 'concurrency must be 1..32') action = Path(args.action_script).expanduser().resolve() check(action.is_file() and os.access(action, os.X_OK), 'action script must be executable') bootstrap = Client(args.base_url, args.ca_file, timeout=5) alice, _, _, chat = bootstrap.bootstrap() barrier = threading.Barrier(args.concurrency + 1) def worker(worker_id): client = Client(args.base_url, args.ca_file, timeout=5) barrier.wait() deadline = time.monotonic() + args.duration latencies = [] errors = 0 instances = set() index = 0 while time.monotonic() < deadline: started = time.monotonic() try: response = client.message( alice, chat, 'rolling-%s-%s' % (worker_id, index)) latencies.append(time.monotonic() - started) instances.add(response['headers']['X-Instance-Id']) except Failure: errors += 1 index += 1 return latencies, errors, instances started = time.monotonic() with ThreadPoolExecutor(max_workers=args.concurrency) as pool: futures = [pool.submit(worker, number) for number in range(args.concurrency)] barrier.wait() time.sleep(3) action_started = time.monotonic() process = subprocess.run( [str(action)], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=args.duration - 5) action_seconds = time.monotonic() - action_started check(process.returncode == 0, 'rolling action failed') results = [future.result() for future in futures] elapsed = time.monotonic() - started latencies = sorted(value for values, _, _ in results for value in values) errors = sum(value for _, value, _ in results) instances = set().union(*(value for _, _, value in results)) total = len(latencies) + errors def percentile(fraction): if not latencies: return None index = max(0, math.ceil(fraction * len(latencies)) - 1) return round(1000 * latencies[index], 3) report = { 'scenario': 'sequential API replacement under closed-loop writes', 'requested_duration_seconds': args.duration, 'elapsed_seconds': round(elapsed, 3), 'rolling_action_seconds': round(action_seconds, 3), 'concurrency': args.concurrency, 'observed_instance_ids': sorted(instances), 'successful_requests': len(latencies), 'failed_requests': errors, 'error_ratio': round(errors / total, 6) if total else None, 'successful_rps': round(len(latencies) / elapsed, 3), 'successful_latency_ms': { 'p50': percentile(.50), 'p95': percentile(.95), 'p99': percentile(.99), }, 'note': 'Closed-loop result on one host; success latency excludes failures.', } output = json.dumps(report, ensure_ascii=False, indent=2) if args.output: destination = Path(args.output) destination.parent.mkdir(parents=True, exist_ok=True) destination.write_text(output + '\n', encoding='utf-8') print(output) check(bool(latencies), 'no successful writes') check(len(instances) >= 3, 'rolling replacement did not expose replacement instance IDs') if __name__ == '__main__': run(main)