"""Concurrent idempotency, both replica failures, and rolling recreation under writes.""" import json import os import subprocess import sys import threading import time import uuid from concurrent.futures import ThreadPoolExecutor from pathlib import Path from urllib.parse import quote ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT / 'tests')) from client import Client, check, Failure def compose(*args): subprocess.run(['docker', 'compose', *args], cwd=ROOT, check=True, stdout=subprocess.DEVNULL, timeout=120) def history(c, actor, chat): path = '/api/v1/chats/' + chat['id'] + '/messages?limit=100' result = [] while path: page = c.call('GET', path, actor=actor, schema='MessagePage')['json'] result.extend(page['items']) path = '/api/v1/chats/' + chat['id'] + '/messages?limit=100&cursor=' + quote(page['next_cursor'], safe='') if page['next_cursor'] else None return result def main(): c = Client(timeout=5) alice, bob, eve, chat = c.bootstrap() key = uuid.uuid4().hex def send(_): return Client().message(alice, chat, 'One logical message', key=key) with ThreadPoolExecutor(max_workers=12) as executor: responses = list(executor.map(send, range(24))) ids = {item['json']['id'] for item in responses} instances = {item['headers']['X-Instance-Id'] for item in responses} check(len(ids) == 1 and len(instances) >= 2, 'Concurrency did not cover both replicas or duplicated message') check(len(history(c, alice, chat)) == 1, 'Duplicate row committed') c.call('POST', '/api/v1/chats/' + chat['id'] + '/messages', 409, actor=alice, headers={'Idempotency-Key': key}, body={'text': 'Different'}) results = [{'scenario': 'concurrent_idempotency', 'requests': 24, 'replicas': sorted(instances), 'unique_messages': 1}] def experiment(label, action): stop = threading.Event() writes, errors, times = [], [], [] def traffic(): client = Client(timeout=3) retry_key, text = uuid.uuid4().hex, label + uuid.uuid4().hex while not stop.is_set(): try: response = client.message(alice, chat, text, key=retry_key) writes.append(response['json']) times.append(time.monotonic()) retry_key, text = uuid.uuid4().hex, label + uuid.uuid4().hex except Failure: errors.append(time.monotonic()) time.sleep(0.02) thread = threading.Thread(target=traffic) thread.start() started = time.monotonic() try: action() time.sleep(5) finally: stop.set() thread.join(timeout=10) duration = time.monotonic() - started stored = {item['id']: item for item in history(c, bob, chat)} check(writes and all(stored.get(item['id']) == item for item in writes), 'Confirmed writes lost') bodies = [item['text'] for item in stored.values()] check(len(bodies) == len(set(bodies)), 'Retry duplicated a logical message') gaps = [b-a for a,b in zip([started]+times, times+[time.monotonic()])] check(max(gaps) < 10, 'Recovery exceeded 10 seconds') results.append({'scenario': label, 'duration_seconds': round(duration,3), 'writes': len(writes), 'errors': len(errors), 'rps': round(len(writes)/duration,3), 'max_success_gap_seconds': round(max(gaps),3)}) for service in ('api1', 'api2'): try: experiment('stop_' + service, lambda: compose('stop', '-t', '2', service)) finally: compose('up', '-d', '--no-deps', '--wait', service) time.sleep(2) def rolling(): for service in ('api1', 'api2'): compose('up', '-d', '--no-deps', '--force-recreate', '--wait', service) time.sleep(2) experiment('rolling_recreation', rolling) Path(ROOT / 'evidence/replication.json').write_text(json.dumps(results, indent=2), encoding='utf-8') print(json.dumps(results, indent=2)) if __name__ == '__main__': main()