102 lines
4.1 KiB
Python
102 lines
4.1 KiB
Python
"""Concurrent idempotency, both replica failures, and rolling recreation under writes."""
|
|
import json
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
import threading
|
|
import time
|
|
import uuid
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
from pathlib import Path
|
|
from urllib.parse import quote
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT / 'tests'))
|
|
from client import Client, check, Failure
|
|
|
|
|
|
def compose(*args):
|
|
subprocess.run(['docker', 'compose', *args], cwd=ROOT, check=True,
|
|
stdout=subprocess.DEVNULL, timeout=120)
|
|
|
|
|
|
def history(c, actor, chat):
|
|
path = '/api/v1/chats/' + chat['id'] + '/messages?limit=100'
|
|
result = []
|
|
while path:
|
|
page = c.call('GET', path, actor=actor, schema='MessagePage')['json']
|
|
result.extend(page['items'])
|
|
path = '/api/v1/chats/' + chat['id'] + '/messages?limit=100&cursor=' + quote(page['next_cursor'], safe='') if page['next_cursor'] else None
|
|
return result
|
|
|
|
|
|
def main():
|
|
c = Client(timeout=5)
|
|
alice, bob, eve, chat = c.bootstrap()
|
|
key = uuid.uuid4().hex
|
|
def send(_):
|
|
return Client().message(alice, chat, 'One logical message', key=key)
|
|
with ThreadPoolExecutor(max_workers=12) as executor:
|
|
responses = list(executor.map(send, range(24)))
|
|
ids = {item['json']['id'] for item in responses}
|
|
instances = {item['headers']['X-Instance-Id'] for item in responses}
|
|
check(len(ids) == 1 and len(instances) >= 2, 'Concurrency did not cover both replicas or duplicated message')
|
|
check(len(history(c, alice, chat)) == 1, 'Duplicate row committed')
|
|
c.call('POST', '/api/v1/chats/' + chat['id'] + '/messages', 409, actor=alice,
|
|
headers={'Idempotency-Key': key}, body={'text': 'Different'})
|
|
results = [{'scenario': 'concurrent_idempotency', 'requests': 24,
|
|
'replicas': sorted(instances), 'unique_messages': 1}]
|
|
|
|
def experiment(label, action):
|
|
stop = threading.Event()
|
|
writes, errors, times = [], [], []
|
|
def traffic():
|
|
client = Client(timeout=3)
|
|
retry_key, text = uuid.uuid4().hex, label + uuid.uuid4().hex
|
|
while not stop.is_set():
|
|
try:
|
|
response = client.message(alice, chat, text, key=retry_key)
|
|
writes.append(response['json'])
|
|
times.append(time.monotonic())
|
|
retry_key, text = uuid.uuid4().hex, label + uuid.uuid4().hex
|
|
except Failure:
|
|
errors.append(time.monotonic())
|
|
time.sleep(0.02)
|
|
thread = threading.Thread(target=traffic)
|
|
thread.start()
|
|
started = time.monotonic()
|
|
try:
|
|
action()
|
|
time.sleep(5)
|
|
finally:
|
|
stop.set()
|
|
thread.join(timeout=10)
|
|
duration = time.monotonic() - started
|
|
stored = {item['id']: item for item in history(c, bob, chat)}
|
|
check(writes and all(stored.get(item['id']) == item for item in writes), 'Confirmed writes lost')
|
|
bodies = [item['text'] for item in stored.values()]
|
|
check(len(bodies) == len(set(bodies)), 'Retry duplicated a logical message')
|
|
gaps = [b-a for a,b in zip([started]+times, times+[time.monotonic()])]
|
|
check(max(gaps) < 10, 'Recovery exceeded 10 seconds')
|
|
results.append({'scenario': label, 'duration_seconds': round(duration,3),
|
|
'writes': len(writes), 'errors': len(errors),
|
|
'rps': round(len(writes)/duration,3), 'max_success_gap_seconds': round(max(gaps),3)})
|
|
|
|
for service in ('api1', 'api2'):
|
|
try:
|
|
experiment('stop_' + service, lambda: compose('stop', '-t', '2', service))
|
|
finally:
|
|
compose('up', '-d', '--no-deps', '--wait', service)
|
|
time.sleep(2)
|
|
def rolling():
|
|
for service in ('api1', 'api2'):
|
|
compose('up', '-d', '--no-deps', '--force-recreate', '--wait', service)
|
|
time.sleep(2)
|
|
experiment('rolling_recreation', rolling)
|
|
Path(ROOT / 'evidence/replication.json').write_text(json.dumps(results, indent=2), encoding='utf-8')
|
|
print(json.dumps(results, indent=2))
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|