"""Run concurrent writes while both API replicas are replaced one at a time.""" import concurrent.futures import json import math import subprocess import sys import threading import time import uuid from pathlib import Path ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT / "tests")) from client import Client, Failure, arguments, check, run # noqa: E402 def main(): args = arguments("Rolling update under write load") base = Client(args.base_url, args.ca_file) alice, _, _, chat = base.bootstrap() path = "/api/v1/chats/" + chat["id"] + "/messages" workers = 4 start_barrier = threading.Barrier(workers + 1) stop = threading.Event() def worker(worker_id): client = Client(args.base_url, args.ca_file, timeout=3) successes = failures = 0 latencies = [] instances = set() index = 0 start_barrier.wait() while not stop.is_set(): key = uuid.uuid4().hex started = time.monotonic() try: result = client.call("POST", path, 201, actor=alice, body={"text": "rolling-%d-%d" % (worker_id, index)}, headers={"Idempotency-Key": key}, schema="Message") successes += 1 latencies.append(time.monotonic() - started) instances.add(result["headers"]["X-Instance-Id"]) except Failure: failures += 1 index += 1 return successes, failures, latencies, instances started = time.monotonic() with concurrent.futures.ThreadPoolExecutor(max_workers=workers) as pool: futures = [pool.submit(worker, i) for i in range(workers)] start_barrier.wait() update = subprocess.run([str(ROOT / "scripts" / "rolling-update.sh")], cwd=ROOT, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, text=True, timeout=120) time.sleep(3) stop.set() results = [future.result() for future in futures] elapsed = time.monotonic() - started check(update.returncode == 0, "rolling update failed: " + update.stderr[-500:]) successes = sum(item[0] for item in results) failures = sum(item[1] for item in results) latencies = sorted(value for item in results for value in item[2]) instances = sorted(set().union(*(item[3] for item in results))) check(successes > 0, "no successful writes during rolling update") check(set(instances) == {"api-1", "api-2"}, "both replicas were not observed during rolling update") def percentile(fraction): if not latencies: return None pos = max(0, math.ceil(fraction * len(latencies)) - 1) return round(latencies[pos] * 1000, 3) output = { "workload": "four-client closed-loop POST messages", "elapsed_seconds": round(elapsed, 3), "successful_requests": successes, "failed_requests": failures, "successful_rps": round(successes / elapsed, 3), "latency_ms_successes": {"p50": percentile(.5), "p95": percentile(.95), "p99": percentile(.99)}, "instances": instances, } (ROOT / "evidence").mkdir(exist_ok=True) (ROOT / "evidence" / "rolling-update.json").write_text( json.dumps(output, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") print("PASS:", json.dumps(output, ensure_ascii=False)) if __name__ == "__main__": run(main)